You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/18 01:30:48 UTC
[01/34] git commit: HDFS-6924. Add new RAM_DISK storage type. (Arpit
Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 db2bfed83 -> 9183dadb7
HDFS-6924. Add new RAM_DISK storage type. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d09f3165
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d09f3165
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d09f3165
Branch: refs/heads/branch-2.6
Commit: d09f3165200be95af7076495fd1719ec209af061
Parents: db2bfed
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 09:03:45 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:34 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/StorageType.java | 3 ++-
.../java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 ++++
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto | 2 +-
.../org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java | 9 ++++++---
.../apache/hadoop/hdfs/server/datanode/TestDataDirs.java | 5 ++++-
5 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d09f3165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index 7ca9e00..c11d4e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -33,7 +33,8 @@ import org.apache.hadoop.classification.InterfaceStability;
public enum StorageType {
DISK,
SSD,
- ARCHIVE;
+ ARCHIVE,
+ RAM_DISK;
public static final StorageType DEFAULT = DISK;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d09f3165/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index d26e147..8e5e8cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -1785,6 +1785,8 @@ public class PBHelper {
return StorageTypeProto.SSD;
case ARCHIVE:
return StorageTypeProto.ARCHIVE;
+ case RAM_DISK:
+ return StorageTypeProto.RAM_DISK;
default:
throw new IllegalStateException(
"BUG: StorageType not found, type=" + type);
@@ -1815,6 +1817,8 @@ public class PBHelper {
return StorageType.SSD;
case ARCHIVE:
return StorageType.ARCHIVE;
+ case RAM_DISK:
+ return StorageType.RAM_DISK;
default:
throw new IllegalStateException(
"BUG: StorageTypeProto not found, type=" + type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d09f3165/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index 38e649e..10af3b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -159,6 +159,7 @@ enum StorageTypeProto {
DISK = 1;
SSD = 2;
ARCHIVE = 3;
+ RAM_DISK = 4;
}
/**
@@ -305,7 +306,6 @@ message HdfsFileStatusProto {
// Optional field for fileId
optional uint64 fileId = 13 [default = 0]; // default as an invalid id
optional int32 childrenNum = 14 [default = -1];
-
// Optional field for file encryption
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d09f3165/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index cb85c7d..98fd59a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -448,13 +448,16 @@ public class TestPBHelper {
DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
AdminStates.DECOMMISSIONED),
DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3",
- AdminStates.NORMAL)
+ AdminStates.NORMAL),
+ DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h4",
+ AdminStates.NORMAL),
};
- String[] storageIDs = {"s1", "s2", "s3"};
+ String[] storageIDs = {"s1", "s2", "s3", "s4"};
StorageType[] media = {
StorageType.DISK,
StorageType.SSD,
- StorageType.DISK
+ StorageType.DISK,
+ StorageType.RAM_DISK
};
LocatedBlock lb = new LocatedBlock(
new ExtendedBlock("bp12", 12345, 10, 53),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d09f3165/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
index 53babb4..c0b4f9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
@@ -44,10 +44,11 @@ public class TestDataDirs {
File dir1 = new File("/dir1");
File dir2 = new File("/dir2");
File dir3 = new File("/dir3");
+ File dir4 = new File("/dir4");
// Verify that a valid string is correctly parsed, and that storage
// type is not case-sensitive
- String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3";
+ String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3,[ram_disk]/dir4";
conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1);
locations = DataNode.getStorageLocations(conf);
assertThat(locations.size(), is(4));
@@ -59,6 +60,8 @@ public class TestDataDirs {
assertThat(locations.get(2).getUri(), is(dir2.toURI()));
assertThat(locations.get(3).getStorageType(), is(StorageType.DISK));
assertThat(locations.get(3).getUri(), is(dir3.toURI()));
+ assertThat(locations.get(4).getStorageType(), is(StorageType.RAM_DISK));
+ assertThat(locations.get(4).getUri(), is(dir4.toURI()));
// Verify that an unrecognized storage type result in an exception.
String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2";
[09/34] git commit: HDFS-7066. LazyWriter#evictBlocks misses a null
check for replicaState. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-7066. LazyWriter#evictBlocks misses a null check for replicaState. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2fdd9f51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2fdd9f51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2fdd9f51
Branch: refs/heads/branch-2.6
Commit: 2fdd9f51cea3d1d8b498632ae85d7c8ee91b9d4e
Parents: bcec658
Author: arp <ar...@apache.org>
Authored: Mon Sep 15 15:28:17 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fdd9f51/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 2b16a65..44a91c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2410,6 +2410,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LazyWriteReplicaTracker.ReplicaState replicaState =
lazyWriteReplicaTracker.getNextCandidateForEviction();
+ if (replicaState == null) {
+ break;
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("Evicting block " + replicaState);
}
[23/34] git commit: HDFS-7155. Bugfix in createLocatedFileStatus
caused by bad merge. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7155. Bugfix in createLocatedFileStatus caused by bad merge. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3481f5d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3481f5d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3481f5d4
Branch: refs/heads/branch-2.6
Commit: 3481f5d4f3d4e15074a8e976ff2dbbf283f39b96
Parents: 05cfd68
Author: arp <ar...@apache.org>
Authored: Fri Sep 26 16:30:47 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3481f5d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c2dfeae..56f9045 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2464,6 +2464,7 @@ public class FSDirectory implements Closeable {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
replication = fileNode.getFileReplication(snapshot);
+ blocksize = fileNode.getPreferredBlockSize();
isLazyPersist = fileNode.getLazyPersistFlag();
final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID;
[17/34] git commit: HDFS-7091. Add forwarding constructor for
INodeFile for existing callers. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7091. Add forwarding constructor for INodeFile for existing callers. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebdbb5ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebdbb5ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebdbb5ec
Branch: refs/heads/branch-2.6
Commit: ebdbb5ec2415d387327aa1b34d41053761f767a1
Parents: d32ce57
Author: arp <ar...@apache.org>
Authored: Thu Sep 18 19:20:25 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/FSImageFormat.java | 2 +-
.../hadoop/hdfs/server/namenode/FSImageSerialization.java | 2 +-
.../org/apache/hadoop/hdfs/server/namenode/INodeFile.java | 7 +++++++
.../apache/hadoop/hdfs/server/namenode/CreateEditsLog.java | 4 ++--
.../org/apache/hadoop/hdfs/server/namenode/TestEditLog.java | 2 +-
.../hadoop/hdfs/server/namenode/TestFSPermissionChecker.java | 2 +-
.../apache/hadoop/hdfs/server/namenode/TestINodeFile.java | 8 ++++----
7 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 6d050e9..fd0f467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -789,7 +789,7 @@ public class FSImageFormat {
// Images in the old format will not have the lazyPersist flag so it is
// safe to pass false always.
final INodeFile file = new INodeFile(inodeId, localName, permissions,
- modificationTime, atime, blocks, replication, blockSize, (byte)0, false);
+ modificationTime, atime, blocks, replication, blockSize, (byte)0);
if (underConstruction) {
file.toUnderConstruction(clientName, clientMachine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 1872898..e2a9a93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -151,7 +151,7 @@ public class FSImageSerialization {
// Images in the pre-protobuf format will not have the lazyPersist flag,
// so it is safe to pass false always.
INodeFile file = new INodeFile(inodeId, name, perm, modificationTime,
- modificationTime, blocks, blockReplication, preferredBlockSize, (byte)0, false);
+ modificationTime, blocks, blockReplication, preferredBlockSize, (byte)0);
file.toUnderConstruction(clientName, clientMachine);
return file;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 05e7bd2..aeddcad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -120,6 +120,13 @@ public class INodeFile extends INodeWithAdditionalFields
private BlockInfo[] blocks;
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
+ long atime, BlockInfo[] blklist, short replication,
+ long preferredBlockSize) {
+ this(id, name, permissions, mtime, atime, blklist, replication,
+ preferredBlockSize, false);
+ }
+
+ INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize, byte storagePolicyID, boolean isLazyPersist) {
super(id, name, permissions, mtime, atime);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index bcce0fd..94b139b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -82,7 +82,7 @@ public class CreateEditsLog {
}
final INodeFile inode = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, blocks, replication, blockSize, (byte)0, false);
+ p, 0L, 0L, blocks, replication, blockSize, (byte)0);
inode.toUnderConstruction("", "");
// Append path to filename with information about blockIDs
@@ -97,7 +97,7 @@ public class CreateEditsLog {
editLog.logMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0, false);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0);
fileUc.toUnderConstruction("", "");
editLog.logOpenFile(filePath, fileUc, false, false);
editLog.logCloseFile(filePath, inode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 2cc9be4..89ef215 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -194,7 +194,7 @@ public class TestEditLog {
for (int i = 0; i < numTransactions; i++) {
INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0, false);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0);
inode.toUnderConstruction("", "");
editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
index 079218a..9bee4a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
@@ -432,7 +432,7 @@ public class TestFSPermissionChecker {
FsPermission.createImmutable(perm));
INodeFile inodeFile = new INodeFile(INodeId.GRANDFATHER_INODE_ID,
name.getBytes("UTF-8"), permStatus, 0L, 0L, null, REPLICATION,
- PREFERRED_BLOCK_SIZE, (byte)0, false);
+ PREFERRED_BLOCK_SIZE, (byte)0);
parent.addChild(inodeFile);
return inodeFile;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdbb5ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index bfaf9d8..d688724 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -85,7 +85,7 @@ public class TestINodeFile {
INodeFile createINodeFile(short replication, long preferredBlockSize) {
return new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, replication, preferredBlockSize, (byte)0, false);
+ null, replication, preferredBlockSize, (byte)0);
}
private static INodeFile createINodeFile(byte storagePolicyID) {
@@ -286,7 +286,7 @@ public class TestINodeFile {
INodeFile[] iNodes = new INodeFile[nCount];
for (int i = 0; i < nCount; i++) {
iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
- preferredBlockSize, (byte)0, false);
+ preferredBlockSize, (byte)0);
iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
BlockInfo newblock = new BlockInfo(replication);
iNodes[i].addBlock(newblock);
@@ -344,7 +344,7 @@ public class TestINodeFile {
{//cast from INodeFileUnderConstruction
final INode from = new INodeFile(
INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, replication,
- 1024L, (byte)0, false);
+ 1024L, (byte)0);
from.asFile().toUnderConstruction("client", "machine");
//cast to INodeFile, should success
@@ -1068,7 +1068,7 @@ public class TestINodeFile {
public void testFileUnderConstruction() {
replication = 3;
final INodeFile file = new INodeFile(INodeId.GRANDFATHER_INODE_ID, null,
- perm, 0L, 0L, null, replication, 1024L, (byte)0, false);
+ perm, 0L, 0L, null, replication, 1024L, (byte)0);
assertFalse(file.isUnderConstruction());
final String clientName = "client";
[21/34] git commit: HDFS-6978. Directory scanner should correctly
reconcile blocks on RAM disk. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6978. Directory scanner should correctly reconcile blocks on RAM disk. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cc206b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cc206b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cc206b8
Branch: refs/heads/branch-2.6
Commit: 6cc206b8adb7f21ea2b2c1cadeeeb98a3d8b3de9
Parents: 2fdd9f5
Author: arp <ar...@apache.org>
Authored: Fri Sep 12 22:13:18 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/DirectoryScanner.java | 19 ++-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 9 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 57 ++++---
.../fsdataset/impl/LazyWriteReplicaTracker.java | 12 +-
.../server/datanode/TestDirectoryScanner.java | 171 +++++++++++++++++--
.../fsdataset/impl/FsDatasetTestUtil.java | 10 ++
.../fsdataset/impl/TestLazyPersistFiles.java | 14 +-
7 files changed, 241 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index c313b04..71f976b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -83,6 +83,7 @@ public class DirectoryScanner implements Runnable {
long missingBlockFile = 0;
long missingMemoryBlocks = 0;
long mismatchBlocks = 0;
+ long duplicateBlocks = 0;
public Stats(String bpid) {
this.bpid = bpid;
@@ -440,7 +441,7 @@ public class DirectoryScanner implements Runnable {
int d = 0; // index for blockpoolReport
int m = 0; // index for memReprot
while (m < memReport.length && d < blockpoolReport.length) {
- Block memBlock = memReport[Math.min(m, memReport.length - 1)];
+ FinalizedReplica memBlock = memReport[Math.min(m, memReport.length - 1)];
ScanInfo info = blockpoolReport[Math.min(
d, blockpoolReport.length - 1)];
if (info.getBlockId() < memBlock.getBlockId()) {
@@ -468,9 +469,23 @@ public class DirectoryScanner implements Runnable {
// or block file length is different than expected
statsRecord.mismatchBlocks++;
addDifference(diffRecord, statsRecord, info);
+ } else if (info.getBlockFile().compareTo(memBlock.getBlockFile()) != 0) {
+ // volumeMap record and on-disk files don't match.
+ statsRecord.duplicateBlocks++;
+ addDifference(diffRecord, statsRecord, info);
}
d++;
- m++;
+
+ if (d < blockpoolReport.length) {
+ // There may be multiple on-disk records for the same block, don't increment
+ // the memory record pointer if so.
+ ScanInfo nextInfo = blockpoolReport[Math.min(d, blockpoolReport.length - 1)];
+ if (nextInfo.getBlockId() != info.blockId) {
+ ++m;
+ }
+ } else {
+ ++m;
+ }
}
while (m < memReport.length) {
FinalizedReplica current = memReport[m++];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index f5c6ed6..c370f3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -269,15 +269,16 @@ class BlockPoolSlice {
/**
* Save the given replica to persistent storage.
*
- * @param replicaInfo
* @return The saved meta and block files, in that order.
* @throws IOException
*/
- File[] lazyPersistReplica(ReplicaInfo replicaInfo) throws IOException {
+ File[] lazyPersistReplica(long blockId, long genStamp,
+ File srcMeta, File srcFile) throws IOException {
if (!lazypersistDir.exists() && !lazypersistDir.mkdirs()) {
FsDatasetImpl.LOG.warn("Failed to create " + lazypersistDir);
}
- File targetFiles[] = FsDatasetImpl.copyBlockFiles(replicaInfo, lazypersistDir);
+ File targetFiles[] = FsDatasetImpl.copyBlockFiles(
+ blockId, genStamp, srcMeta, srcFile, lazypersistDir);
dfsUsage.incDfsUsed(targetFiles[0].length() + targetFiles[1].length());
return targetFiles;
}
@@ -510,7 +511,7 @@ class BlockPoolSlice {
* @return the replica that is retained.
* @throws IOException
*/
- private ReplicaInfo resolveDuplicateReplicas(
+ ReplicaInfo resolveDuplicateReplicas(
final ReplicaInfo replica1, final ReplicaInfo replica2,
final ReplicaMap volumeMap) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 44a91c1..e243852 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -703,17 +703,16 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
/**
- * Copy the block and meta files for the given block from the given
+ * Copy the block and meta files for the given block to the given destination.
* @return the new meta and block files.
* @throws IOException
*/
- static File[] copyBlockFiles(ReplicaInfo replicaInfo, File destRoot)
+ static File[] copyBlockFiles(long blockId, long genStamp,
+ File srcMeta, File srcFile, File destRoot)
throws IOException {
- final File destDir = DatanodeUtil.idToBlockDir(destRoot, replicaInfo.getBlockId());
- final File dstFile = new File(destDir, replicaInfo.getBlockName());
- final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, replicaInfo.getGenerationStamp());
- final File srcMeta = replicaInfo.getMetaFile();
- final File srcFile = replicaInfo.getBlockFile();
+ final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId);
+ final File dstFile = new File(destDir, srcFile.getName());
+ final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp);
try {
FileUtils.copyFile(srcMeta, dstMeta);
} catch (IOException e) {
@@ -1847,10 +1846,20 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File memFile = memBlockInfo.getBlockFile();
if (memFile.exists()) {
if (memFile.compareTo(diskFile) != 0) {
- LOG.warn("Block file " + memFile.getAbsolutePath()
- + " does not match file found by scan "
- + diskFile.getAbsolutePath());
- // TODO: Should the diskFile be deleted?
+ if (diskMetaFile.exists()) {
+ if (memBlockInfo.getMetaFile().exists()) {
+ // We have two sets of block+meta files. Decide which one to
+ // keep.
+ ReplicaInfo diskBlockInfo = new FinalizedReplica(
+ blockId, diskFile.length(), diskGS, vol, diskFile.getParentFile());
+ ((FsVolumeImpl) vol).getBlockPoolSlice(bpid).resolveDuplicateReplicas(
+ memBlockInfo, diskBlockInfo, volumeMap);
+ }
+ } else {
+ if (!diskFile.delete()) {
+ LOG.warn("Failed to delete " + diskFile + ". Will retry on next scan");
+ }
+ }
}
} else {
// Block refers to a block file that does not exist.
@@ -2315,6 +2324,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
FsVolumeImpl targetVolume;
ReplicaInfo replicaInfo;
+ BlockPoolSlice bpSlice;
+ File srcFile, srcMeta;
+ long genStamp;
synchronized (FsDatasetImpl.this) {
replicaInfo = volumeMap.get(bpid, blockId);
@@ -2336,10 +2348,18 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
- File[] savedFiles = targetVolume.getBlockPoolSlice(bpid)
- .lazyPersistReplica(replicaInfo);
- lazyWriteReplicaTracker.recordEndLazyPersist(
- bpid, blockId, savedFiles[0], savedFiles[1]);
+ bpSlice = targetVolume.getBlockPoolSlice(bpid);
+ srcMeta = replicaInfo.getMetaFile();
+ srcFile = replicaInfo.getBlockFile();
+ genStamp = replicaInfo.getGenerationStamp();
+ }
+
+ // Drop the FsDatasetImpl lock for the file copy.
+ File[] savedFiles =
+ bpSlice.lazyPersistReplica(blockId, genStamp, srcMeta, srcFile);
+
+ synchronized (FsDatasetImpl.this) {
+ lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedFiles);
if (LOG.isDebugEnabled()) {
LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
@@ -2360,7 +2380,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
try {
replicaState = lazyWriteReplicaTracker.dequeueNextReplicaToPersist();
if (replicaState != null) {
- // Move the replica outside the lock.
moveReplicaToNewVolume(replicaState.bpid, replicaState.blockId);
}
succeeded = true;
@@ -2459,9 +2478,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Remove the old replicas from transient storage.
if (blockFile.delete() || !blockFile.exists()) {
((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, blockFileUsed);
- }
- if (metaFile.delete() || !metaFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, metaFileUsed);
+ if (metaFile.delete() || !metaFile.exists()) {
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, metaFileUsed);
+ }
}
// If deletion failed then the directory scanner will cleanup the blocks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
index a8ab1b9..e8d9c5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
@@ -161,9 +161,13 @@ class LazyWriteReplicaTracker {
replicaState.lazyPersistVolume = checkpointVolume;
}
+ /**
+ * @param bpid
+ * @param blockId
+ * @param savedFiles The saved meta and block files, in that order.
+ */
synchronized void recordEndLazyPersist(
- final String bpid, final long blockId,
- final File savedMetaFile, final File savedBlockFile) {
+ final String bpid, final long blockId, final File[] savedFiles) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
ReplicaState replicaState = map.get(blockId);
@@ -172,8 +176,8 @@ class LazyWriteReplicaTracker {
bpid + "; blockId=" + blockId);
}
replicaState.state = State.LAZY_PERSIST_COMPLETE;
- replicaState.savedMetaFile = savedMetaFile;
- replicaState.savedBlockFile = savedBlockFile;
+ replicaState.savedMetaFile = savedFiles[0];
+ replicaState.savedBlockFile = savedFiles[1];
if (replicasNotPersisted.peek() == replicaState) {
// Common case.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index 78b9e2b..f27e117 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -31,22 +33,21 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Random;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.*;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
/**
@@ -60,22 +61,29 @@ public class TestDirectoryScanner {
private MiniDFSCluster cluster;
private String bpid;
+ private DFSClient client;
private FsDatasetSpi<? extends FsVolumeSpi> fds = null;
private DirectoryScanner scanner = null;
private final Random rand = new Random();
private final Random r = new Random();
+ private static final int BLOCK_LENGTH = 100;
static {
- CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
+ CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_LENGTH);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
}
/** create a file with a length of <code>fileLen</code> */
- private void createFile(String fileName, long fileLen) throws IOException {
+ private List<LocatedBlock> createFile(String fileNamePrefix,
+ long fileLen,
+ boolean isLazyPersist) throws IOException {
FileSystem fs = cluster.getFileSystem();
- Path filePath = new Path(fileName);
- DFSTestUtil.createFile(fs, filePath, fileLen, (short) 1, r.nextLong());
+ Path filePath = new Path("/" + fileNamePrefix + ".dat");
+ DFSTestUtil.createFile(
+ fs, filePath, isLazyPersist, 1024, fileLen,
+ BLOCK_LENGTH, (short) 1, r.nextLong(), false);
+ return client.getLocatedBlocks(filePath.toString(), 0, fileLen).getLocatedBlocks();
}
/** Truncate a block file */
@@ -134,6 +142,43 @@ public class TestDirectoryScanner {
return 0;
}
+ /**
+ * Duplicate the given block on all volumes.
+ * @param blockId
+ * @throws IOException
+ */
+ private void duplicateBlock(long blockId) throws IOException {
+ synchronized (fds) {
+ ReplicaInfo b = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId);
+ for (FsVolumeSpi v : fds.getVolumes()) {
+ if (v.getStorageID().equals(b.getVolume().getStorageID())) {
+ continue;
+ }
+
+ // Volume without a copy of the block. Make a copy now.
+ File sourceBlock = b.getBlockFile();
+ File sourceMeta = b.getMetaFile();
+ String sourceRoot = b.getVolume().getBasePath();
+ String destRoot = v.getBasePath();
+
+ String relativeBlockPath = new File(sourceRoot).toURI().relativize(sourceBlock.toURI()).getPath();
+ String relativeMetaPath = new File(sourceRoot).toURI().relativize(sourceMeta.toURI()).getPath();
+
+ File destBlock = new File(destRoot, relativeBlockPath);
+ File destMeta = new File(destRoot, relativeMetaPath);
+
+ destBlock.getParentFile().mkdirs();
+ FileUtils.copyFile(sourceBlock, destBlock);
+ FileUtils.copyFile(sourceMeta, destMeta);
+
+ if (destBlock.exists() && destMeta.exists()) {
+ LOG.info("Copied " + sourceBlock + " ==> " + destBlock);
+ LOG.info("Copied " + sourceMeta + " ==> " + destMeta);
+ }
+ }
+ }
+ }
+
/** Get a random blockId that is not used already */
private long getFreeBlockId() {
long id = rand.nextLong();
@@ -216,6 +261,12 @@ public class TestDirectoryScanner {
private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile,
long missingMemoryBlocks, long mismatchBlocks) throws IOException {
+ scan(totalBlocks, diffsize, missingMetaFile, missingBlockFile,
+ missingMemoryBlocks, mismatchBlocks, 0);
+ }
+
+ private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile,
+ long missingMemoryBlocks, long mismatchBlocks, long duplicateBlocks) throws IOException {
scanner.reconcile();
assertTrue(scanner.diffs.containsKey(bpid));
@@ -229,9 +280,92 @@ public class TestDirectoryScanner {
assertEquals(missingBlockFile, stats.missingBlockFile);
assertEquals(missingMemoryBlocks, stats.missingMemoryBlocks);
assertEquals(mismatchBlocks, stats.mismatchBlocks);
+ assertEquals(duplicateBlocks, stats.duplicateBlocks);
+ }
+
+ @Test (timeout=300000)
+ public void testRetainBlockOnPersistentStorage() throws Exception {
+ cluster = new MiniDFSCluster
+ .Builder(CONF)
+ .storageTypes(new StorageType[] { StorageType.RAM_DISK, StorageType.DEFAULT })
+ .numDataNodes(1)
+ .build();
+ try {
+ cluster.waitActive();
+ bpid = cluster.getNamesystem().getBlockPoolId();
+ fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
+ client = cluster.getFileSystem().getClient();
+ scanner = new DirectoryScanner(fds, CONF);
+ scanner.setRetainDiffs(true);
+ FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
+
+ // Add a file with 1 block
+ List<LocatedBlock> blocks =
+ createFile(GenericTestUtils.getMethodName(), BLOCK_LENGTH, false);
+
+ // Ensure no difference between volumeMap and disk.
+ scan(1, 0, 0, 0, 0, 0);
+
+ // Make a copy of the block on RAM_DISK and ensure that it is
+ // picked up by the scanner.
+ duplicateBlock(blocks.get(0).getBlock().getBlockId());
+ scan(2, 1, 0, 0, 0, 0, 1);
+ verifyStorageType(blocks.get(0).getBlock().getBlockId(), false);
+ scan(1, 0, 0, 0, 0, 0);
+
+ } finally {
+ if (scanner != null) {
+ scanner.shutdown();
+ scanner = null;
+ }
+ cluster.shutdown();
+ cluster = null;
+ }
}
- @Test
+ @Test (timeout=300000)
+ public void testDeleteBlockOnTransientStorage() throws Exception {
+ cluster = new MiniDFSCluster
+ .Builder(CONF)
+ .storageTypes(new StorageType[] { StorageType.RAM_DISK, StorageType.DEFAULT })
+ .numDataNodes(1)
+ .build();
+ try {
+ cluster.waitActive();
+ bpid = cluster.getNamesystem().getBlockPoolId();
+ fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
+ client = cluster.getFileSystem().getClient();
+ scanner = new DirectoryScanner(fds, CONF);
+ scanner.setRetainDiffs(true);
+ FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
+
+ // Create a file file on RAM_DISK
+ List<LocatedBlock> blocks =
+ createFile(GenericTestUtils.getMethodName(), BLOCK_LENGTH, true);
+
+ // Ensure no difference between volumeMap and disk.
+ scan(1, 0, 0, 0, 0, 0);
+
+ // Make a copy of the block on DEFAULT storage and ensure that it is
+ // picked up by the scanner.
+ duplicateBlock(blocks.get(0).getBlock().getBlockId());
+ scan(2, 1, 0, 0, 0, 0, 1);
+
+ // Ensure that the copy on RAM_DISK was deleted.
+ verifyStorageType(blocks.get(0).getBlock().getBlockId(), false);
+ scan(1, 0, 0, 0, 0, 0);
+
+ } finally {
+ if (scanner != null) {
+ scanner.shutdown();
+ scanner = null;
+ }
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test (timeout=600000)
public void testDirectoryScanner() throws Exception {
// Run the test with and without parallel scanning
for (int parallelism = 1; parallelism < 3; parallelism++) {
@@ -245,16 +379,17 @@ public class TestDirectoryScanner {
cluster.waitActive();
bpid = cluster.getNamesystem().getBlockPoolId();
fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
+ client = cluster.getFileSystem().getClient();
CONF.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY,
parallelism);
scanner = new DirectoryScanner(fds, CONF);
scanner.setRetainDiffs(true);
// Add files with 100 blocks
- createFile("/tmp/t1", 10000);
+ createFile(GenericTestUtils.getMethodName(), BLOCK_LENGTH * 100, false);
long totalBlocks = 100;
- // Test1: No difference between in-memory and disk
+ // Test1: No difference between volumeMap and disk
scan(100, 0, 0, 0, 0, 0);
// Test2: block metafile is missing
@@ -355,7 +490,10 @@ public class TestDirectoryScanner {
assertFalse(scanner.getRunStatus());
} finally {
- scanner.shutdown();
+ if (scanner != null) {
+ scanner.shutdown();
+ scanner = null;
+ }
cluster.shutdown();
}
}
@@ -389,6 +527,13 @@ public class TestDirectoryScanner {
assertEquals(genStamp, memBlock.getGenerationStamp());
}
+ private void verifyStorageType(long blockId, boolean expectTransient) {
+ final ReplicaInfo memBlock;
+ memBlock = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId);
+ assertNotNull(memBlock);
+ assertThat(memBlock.getVolume().isTransientStorage(), is(expectTransient));
+ }
+
private static class TestFsVolumeSpi implements FsVolumeSpi {
@Override
public String[] getBlockPoolList() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
index d6d7dd7..48ddcc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -62,4 +63,13 @@ public class FsDatasetTestUtil {
String bpid) {
return ((FsDatasetImpl)fsd).volumeMap.replicas(bpid);
}
+
+ /**
+ * Stop the lazy writer daemon that saves RAM disk files to persistent storage.
+ * @param dn
+ */
+ public static void stopLazyWriter(DataNode dn) {
+ FsDatasetImpl fsDataset = ((FsDatasetImpl) dn.getFSDataset());
+ ((FsDatasetImpl.LazyWriter) fsDataset.lazyWriter.getRunnable()).stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc206b8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index fcc4798..777779f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -368,6 +368,8 @@ public class TestLazyPersistFiles {
// Found a persisted copy for this block!
boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
assertThat(added, is(true));
+ } else {
+ LOG.error(blockFile + " not found");
}
}
}
@@ -423,7 +425,7 @@ public class TestLazyPersistFiles {
final int SEED = 0XFADED;
// Stop lazy writer to ensure block for path1 is not persisted to disk.
- stopLazyWriter(cluster.getDataNodes().get(0));
+ FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
ensureFileReplicasOnStorageType(path1, RAM_DISK);
@@ -488,7 +490,7 @@ public class TestLazyPersistFiles {
throws IOException, InterruptedException {
startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
- stopLazyWriter(cluster.getDataNodes().get(0));
+ FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
Path path = new Path("/" + METHOD_NAME + ".dat");
makeTestFile(path, BLOCK_SIZE, true);
@@ -682,7 +684,7 @@ public class TestLazyPersistFiles {
throws IOException, InterruptedException {
startUpCluster(true, 1);
- stopLazyWriter(cluster.getDataNodes().get(0));
+ FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -794,12 +796,6 @@ public class TestLazyPersistFiles {
return locatedBlocks;
}
- private void stopLazyWriter(DataNode dn) {
- // Stop the lazyWriter daemon.
- FsDatasetImpl fsDataset = ((FsDatasetImpl) dn.getFSDataset());
- ((FsDatasetImpl.LazyWriter) fsDataset.lazyWriter.getRunnable()).stop();
- }
-
private void makeRandomTestFile(Path path, long length, final boolean isLazyPersist,
long seed) throws IOException {
DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
[03/34] git commit: HDFS-6929. NN periodically unlinks lazy persist
files with missing replicas from namespace. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6929. NN periodically unlinks lazy persist files with missing replicas from namespace. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15372df4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15372df4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15372df4
Branch: refs/heads/branch-2.6
Commit: 15372df48d0f4324b0ea767cde3a2a5e9e29e363
Parents: c2721ed
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:59:31 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 103 ++++++++++++++++++-
.../src/main/resources/hdfs-default.xml | 10 ++
.../fsdataset/impl/TestLazyPersistFiles.java | 46 +++++++++
3 files changed, 158 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15372df4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index fdf4f2e..7dd3874 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -60,6 +60,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROL
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
@@ -436,6 +438,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
Daemon nnrmthread = null; // NamenodeResourceMonitor thread
Daemon nnEditLogRoller = null; // NameNodeEditLogRoller thread
+
+ // A daemon to periodically clean up corrupt lazyPersist files
+ // from the name space.
+ Daemon lazyPersistFileScrubber = null;
/**
* When an active namenode will roll its own edit log, in # edits
*/
@@ -445,6 +451,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
*/
private final int editLogRollerInterval;
+ /**
+ * How frequently we scan and unlink corrupt lazyPersist files.
+ * (In seconds)
+ */
+ private final int lazyPersistFileScrubIntervalSec;
+
private volatile boolean hasResourcesAvailable = false;
private volatile boolean fsRunning = true;
@@ -862,6 +874,15 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS_DEFAULT);
this.inodeId = new INodeId();
+ this.lazyPersistFileScrubIntervalSec = conf.getInt(
+ DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC,
+ DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT);
+
+ if (this.lazyPersistFileScrubIntervalSec == 0) {
+ throw new IllegalArgumentException(
+ DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC + " must be non-zero.");
+ }
+
// For testing purposes, allow the DT secret manager to be started regardless
// of whether security is enabled.
alwaysUseDelegationTokensForTests = conf.getBoolean(
@@ -931,7 +952,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
@VisibleForTesting
static RetryCache initRetryCache(Configuration conf) {
boolean enable = conf.getBoolean(DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY,
- DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT);
+ DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT);
LOG.info("Retry cache on namenode is " + (enable ? "enabled" : "disabled"));
if (enable) {
float heapPercent = conf.getFloat(
@@ -1175,6 +1196,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
editLogRollerThreshold, editLogRollerInterval));
nnEditLogRoller.start();
+ if (lazyPersistFileScrubIntervalSec > 0) {
+ lazyPersistFileScrubber = new Daemon(new LazyPersistFileScrubber(
+ lazyPersistFileScrubIntervalSec));
+ lazyPersistFileScrubber.start();
+ }
+
cacheManager.startMonitorThread();
blockManager.getDatanodeManager().setShouldSendCachingCommands(true);
} finally {
@@ -1228,6 +1255,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
((NameNodeEditLogRoller)nnEditLogRoller.getRunnable()).stop();
nnEditLogRoller.interrupt();
}
+ if (lazyPersistFileScrubber != null) {
+ ((LazyPersistFileScrubber) lazyPersistFileScrubber.getRunnable()).stop();
+ lazyPersistFileScrubber.interrupt();
+ }
if (dir != null && getFSImage() != null) {
if (getFSImage().editLog != null) {
getFSImage().editLog.close();
@@ -2778,6 +2809,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
+ src + " for client " + clientMachine);
}
INodeFile myFile = INodeFile.valueOf(inode, src, true);
+
+ if (myFile.getLazyPersistFlag()) {
+ throw new UnsupportedOperationException(
+ "Cannot append to lazy persist file " + src);
+ }
// Opening an existing file for write - may need to recover lease.
recoverLeaseInternal(myFile, src, holder, clientMachine, false);
@@ -5111,6 +5147,71 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
}
+ /**
+ * Daemon to periodically scan the namespace for lazyPersist files
+ * with missing blocks and unlink them.
+ */
+ class LazyPersistFileScrubber implements Runnable {
+ private volatile boolean shouldRun = true;
+ final int scrubIntervalSec;
+ public LazyPersistFileScrubber(final int scrubIntervalSec) {
+ this.scrubIntervalSec = scrubIntervalSec;
+ }
+
+ /**
+ * Periodically go over the list of lazyPersist files with missing
+ * blocks and unlink them from the namespace.
+ */
+ private void clearCorruptLazyPersistFiles()
+ throws SafeModeException, AccessControlException,
+ UnresolvedLinkException, IOException {
+
+ List<BlockCollection> filesToDelete = new ArrayList<BlockCollection>();
+
+ writeLock();
+
+ try {
+ final Iterator<Block> it = blockManager.getCorruptReplicaBlockIterator();
+
+ while (it.hasNext()) {
+ Block b = it.next();
+ BlockInfo blockInfo = blockManager.getStoredBlock(b);
+ if (blockInfo.getBlockCollection().getLazyPersistFlag()) {
+ filesToDelete.add(blockInfo.getBlockCollection());
+ }
+ }
+
+ for (BlockCollection bc : filesToDelete) {
+ LOG.warn("Removing lazyPersist file " + bc.getName() + " with no replicas.");
+ deleteInternal(bc.getName(), false, false, false);
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ @Override
+ public void run() {
+ while (fsRunning && shouldRun) {
+ try {
+ clearCorruptLazyPersistFiles();
+ Thread.sleep(scrubIntervalSec * 1000);
+ } catch (InterruptedException e) {
+ FSNamesystem.LOG.info(
+ "LazyPersistFileScrubber was interrupted, exiting");
+ break;
+ } catch (Exception e) {
+ FSNamesystem.LOG.error(
+ "Ignoring exception in LazyPersistFileScrubber:", e);
+ }
+ }
+ }
+
+ public void stop() {
+ shouldRun = false;
+ }
+ }
+
public FSImage getFSImage() {
return fsImage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15372df4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 629a7c7..d219336 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -398,6 +398,16 @@
</property>
<property>
+ <name>dfs.namenode.lazypersist.file.scrub.interval.sec</name>
+ <value>300</value>
+ <description>
+ The NameNode periodically scans the namespace for LazyPersist files with
+ missing blocks and unlinks them from the namespace. This configuration key
+ controls the interval between successive scans. Set it to a negative value
+ to disable this behavior.
+ </description>
+</property>
+<property>
<name>dfs.block.access.token.enable</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15372df4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 6e7c091..ddd71b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -204,6 +204,52 @@ public class TestLazyPersistFiles {
}
}
+ /**
+ * If one or more replicas of a lazyPersist file are lost, then the file
+ * must be discarded by the NN, instead of being kept around as a
+ * 'corrupt' file.
+ */
+ @Test (timeout=300000)
+ public void testLazyPersistFilesAreDiscarded()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR,
+ new StorageType[] {RAM_DISK, DEFAULT },
+ (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+
+ makeTestFile(path1, BLOCK_SIZE, true);
+ makeTestFile(path2, BLOCK_SIZE, false);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+ ensureFileReplicasOnStorageType(path2, DEFAULT);
+
+ // Stop the DataNode and sleep for the time it takes the NN to
+ // detect the DN as being dead.
+ cluster.shutdownDataNodes();
+ Thread.sleep(30000L);
+ assertThat(cluster.getNamesystem().getNumDeadDataNodes(), is(1));
+
+ // Next, wait for the replication monitor to mark the file as
+ // corrupt, plus some delta.
+ Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000);
+
+ // Wait for the LazyPersistFileScrubber to run, plus some delta.
+ Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000);
+
+ // Ensure that path1 does not exist anymore, whereas path2 does.
+ assert(!fs.exists(path1));
+ assert(fs.exists(path2));
+
+ // We should have only one block that needs replication i.e. the one
+ // belonging to path2.
+ assertThat(cluster.getNameNode()
+ .getNamesystem()
+ .getBlockManager()
+ .getUnderReplicatedBlocksCount(),
+ is(1L));
+ }
+
@Test (timeout=300000)
public void testLazyPersistBlocksAreSaved()
throws IOException, InterruptedException {
[18/34] git commit: HDFS-7108. Fix unit test failures in
SimulatedFsDataset. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7108. Fix unit test failures in SimulatedFsDataset. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14419889
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14419889
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14419889
Branch: refs/heads/branch-2.6
Commit: 144198892fcac06544c55b6b1de01a7961363588
Parents: ebdbb5e
Author: arp <ar...@apache.org>
Authored: Sat Sep 20 16:12:17 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../server/datanode/SimulatedFSDataset.java | 60 +++++++++++++++++++-
1 file changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14419889/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 312da1f..d1284fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -420,9 +420,66 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
}
+ static class SimulatedVolume implements FsVolumeSpi {
+ private final SimulatedStorage storage;
+
+ SimulatedVolume(final SimulatedStorage storage) {
+ this.storage = storage;
+ }
+
+ @Override
+ public String getStorageID() {
+ return storage.getStorageUuid();
+ }
+
+ @Override
+ public String[] getBlockPoolList() {
+ return new String[0];
+ }
+
+ @Override
+ public long getAvailable() throws IOException {
+ return storage.getCapacity() - storage.getUsed();
+ }
+
+ @Override
+ public String getBasePath() {
+ return null;
+ }
+
+ @Override
+ public String getPath(String bpid) throws IOException {
+ return null;
+ }
+
+ @Override
+ public File getFinalizedDir(String bpid) throws IOException {
+ return null;
+ }
+
+ @Override
+ public StorageType getStorageType() {
+ return null;
+ }
+
+ @Override
+ public boolean isTransientStorage() {
+ return false;
+ }
+
+ @Override
+ public void reserveSpaceForRbw(long bytesToReserve) {
+ }
+
+ @Override
+ public void releaseReservedSpace(long bytesToRelease) {
+ }
+ }
+
private final Map<String, Map<Block, BInfo>> blockMap
= new HashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
+ private final SimulatedVolume volume;
private final String datanodeUuid;
public SimulatedFSDataset(DataStorage storage, Configuration conf) {
@@ -439,6 +496,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
this.storage = new SimulatedStorage(
conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY),
conf.getEnum(CONFIG_PROPERTY_STATE, DEFAULT_STATE));
+ this.volume = new SimulatedVolume(this.storage);
}
public synchronized void injectBlocks(String bpid,
@@ -1138,7 +1196,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override
public FsVolumeSpi getVolume(ExtendedBlock b) {
- throw new UnsupportedOperationException();
+ return volume;
}
@Override
[08/34] git commit: HDFS-6923. Propagate LazyPersist flag to DNs via
DataTransferProtocol. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6923. Propagate LazyPersist flag to DNs via DataTransferProtocol. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c3c0ec9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c3c0ec9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c3c0ec9
Branch: refs/heads/branch-2.6
Commit: 8c3c0ec97763752c2c862d592becf2ee20197d8a
Parents: 1d9d29a
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:13:20 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSOutputStream.java | 13 ++++++++-----
.../protocol/datatransfer/DataTransferProtocol.java | 4 ++--
.../hadoop/hdfs/protocol/datatransfer/Receiver.java | 3 ++-
.../hadoop/hdfs/protocol/datatransfer/Sender.java | 6 ++++--
.../apache/hadoop/hdfs/server/datanode/DataNode.java | 3 ++-
.../hadoop/hdfs/server/datanode/DataXceiver.java | 6 ++++--
.../hadoop-hdfs/src/main/proto/datatransfer.proto | 7 +++++++
.../apache/hadoop/hdfs/TestDataTransferProtocol.java | 2 +-
.../hadoop/hdfs/server/datanode/TestDiskError.java | 2 +-
9 files changed, 31 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index fbe8171..2bcda8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -360,6 +360,7 @@ public class DFSOutputStream extends FSOutputSummer
private long restartDeadline = 0; // Deadline of DN restart
private BlockConstructionStage stage; // block construction stage
private long bytesSent = 0; // number of bytes that've been sent
+ private final boolean isLazyPersistFile;
/** Nodes have been used in the pipeline before and have failed. */
private final List<DatanodeInfo> failed = new ArrayList<DatanodeInfo>();
@@ -377,14 +378,15 @@ public class DFSOutputStream extends FSOutputSummer
* Default construction for file create
*/
private DataStreamer() {
- this(null);
+ this(null, null);
}
/**
* construction with tracing info
*/
- private DataStreamer(Span span) {
+ private DataStreamer(HdfsFileStatus stat, Span span) {
isAppend = false;
+ isLazyPersistFile = stat.isLazyPersist();
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
traceSpan = span;
}
@@ -404,6 +406,7 @@ public class DFSOutputStream extends FSOutputSummer
block = lastBlock.getBlock();
bytesSent = block.getNumBytes();
accessToken = lastBlock.getBlockToken();
+ isLazyPersistFile = stat.isLazyPersist();
long usedInLastBlock = stat.getLen() % blockSize;
int freeInLastBlock = (int)(blockSize - usedInLastBlock);
@@ -1396,7 +1399,7 @@ public class DFSOutputStream extends FSOutputSummer
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
nodes.length, block.getNumBytes(), bytesSent, newGS, checksum,
- cachingStrategy.get());
+ cachingStrategy.get(), isLazyPersistFile);
// receive ack for connect
BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(
@@ -1649,7 +1652,7 @@ public class DFSOutputStream extends FSOutputSummer
if (Trace.isTracing()) {
traceSpan = Trace.startSpan(this.getClass().getSimpleName()).detach();
}
- streamer = new DataStreamer(traceSpan);
+ streamer = new DataStreamer(stat, traceSpan);
if (favoredNodes != null && favoredNodes.length != 0) {
streamer.setFavoredNodes(favoredNodes);
}
@@ -1726,7 +1729,7 @@ public class DFSOutputStream extends FSOutputSummer
} else {
computePacketChunkSize(dfsClient.getConf().writePacketSize,
checksum.getBytesPerChecksum());
- streamer = new DataStreamer(traceSpan);
+ streamer = new DataStreamer(stat, traceSpan);
}
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
index d54d5be..f6b99e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
@@ -106,8 +106,8 @@ public interface DataTransferProtocol {
final long maxBytesRcvd,
final long latestGenerationStamp,
final DataChecksum requestedChecksum,
- final CachingStrategy cachingStrategy) throws IOException;
-
+ final CachingStrategy cachingStrategy,
+ final boolean allowLazyPersist) throws IOException;
/**
* Transfer a block to another datanode.
* The block stage must be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index daae9b7..538c82d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -148,7 +148,8 @@ public abstract class Receiver implements DataTransferProtocol {
fromProto(proto.getRequestedChecksum()),
(proto.hasCachingStrategy() ?
getCachingStrategy(proto.getCachingStrategy()) :
- CachingStrategy.newDefaultStrategy()));
+ CachingStrategy.newDefaultStrategy()),
+ (proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false));
} finally {
if (traceScope != null) traceScope.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index fb6cf2c..1ae9da5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -128,7 +128,8 @@ public class Sender implements DataTransferProtocol {
final long maxBytesRcvd,
final long latestGenerationStamp,
DataChecksum requestedChecksum,
- final CachingStrategy cachingStrategy) throws IOException {
+ final CachingStrategy cachingStrategy,
+ final boolean allowLazyPersist) throws IOException {
ClientOperationHeaderProto header = DataTransferProtoUtil.buildClientHeader(
blk, clientName, blockToken);
@@ -146,7 +147,8 @@ public class Sender implements DataTransferProtocol {
.setMaxBytesRcvd(maxBytesRcvd)
.setLatestGenerationStamp(latestGenerationStamp)
.setRequestedChecksum(checksumProto)
- .setCachingStrategy(getCachingStrategy(cachingStrategy));
+ .setCachingStrategy(getCachingStrategy(cachingStrategy))
+ .setAllowLazyPersist(allowLazyPersist);
if (source != null) {
proto.setSource(PBHelper.convertDatanodeInfo(source));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 8337102..a4db56a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2010,7 +2010,8 @@ public class DataNode extends ReconfigurableBase
new Sender(out).writeBlock(b, targetStorageTypes[0], accessToken,
clientname, targets, targetStorageTypes, srcNode,
- stage, 0, 0, 0, 0, blockSender.getChecksum(), cachingStrategy);
+ stage, 0, 0, 0, 0, blockSender.getChecksum(), cachingStrategy,
+ false);
// send data & checksum
blockSender.sendBlock(out, unbufOut, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 4575c93..3b8304e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -544,7 +544,8 @@ class DataXceiver extends Receiver implements Runnable {
final long maxBytesRcvd,
final long latestGenerationStamp,
DataChecksum requestedChecksum,
- CachingStrategy cachingStrategy) throws IOException {
+ CachingStrategy cachingStrategy,
+ final boolean allowLazyPersist) throws IOException {
previousOpClientName = clientname;
updateCurrentThreadName("Receiving block " + block);
final boolean isDatanode = clientname.length() == 0;
@@ -648,10 +649,11 @@ class DataXceiver extends Receiver implements Runnable {
HdfsConstants.SMALL_BUFFER_SIZE));
mirrorIn = new DataInputStream(unbufMirrorIn);
+ // Do not propagate allowLazyPersist to downstream DataNodes.
new Sender(mirrorOut).writeBlock(originalBlock, targetStorageTypes[0],
blockToken, clientname, targets, targetStorageTypes, srcDataNode,
stage, pipelineSize, minBytesRcvd, maxBytesRcvd,
- latestGenerationStamp, requestedChecksum, cachingStrategy);
+ latestGenerationStamp, requestedChecksum, cachingStrategy, false);
mirrorOut.flush();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
index 098d10a..fb774b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
@@ -115,6 +115,13 @@ message OpWriteBlockProto {
optional CachingStrategyProto cachingStrategy = 10;
optional StorageTypeProto storageType = 11 [default = DISK];
repeated StorageTypeProto targetStorageTypes = 12;
+
+ /**
+ * Hint to the DataNode that the block can be allocated on transient
+ * storage i.e. memory and written to disk lazily. The DataNode is free
+ * to ignore this hint.
+ */
+ optional bool allowLazyPersist = 13 [default = false];
}
message OpTransferBlockProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index 494dd1d..6405b5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -530,6 +530,6 @@ public class TestDataTransferProtocol {
BlockTokenSecretManager.DUMMY_TOKEN, "cl",
new DatanodeInfo[1], new StorageType[1], null, stage,
0, block.getNumBytes(), block.getNumBytes(), newGS,
- checksum, CachingStrategy.newDefaultStrategy());
+ checksum, CachingStrategy.newDefaultStrategy(), false);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c3c0ec9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
index 4b5b6e1..f440bb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
@@ -152,7 +152,7 @@ public class TestDiskError {
BlockTokenSecretManager.DUMMY_TOKEN, "",
new DatanodeInfo[0], new StorageType[0], null,
BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L,
- checksum, CachingStrategy.newDefaultStrategy());
+ checksum, CachingStrategy.newDefaultStrategy(), false);
out.flush();
// close the connection before sending the content of the block
[15/34] git commit: HDFS-6991. Notify NN of evicted block before
deleting it from RAM disk. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6991. Notify NN of evicted block before deleting it from RAM disk. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcec6582
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcec6582
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcec6582
Branch: refs/heads/branch-2.6
Commit: bcec6582cf9ae1411d31b2a6cf97c2169953f5fe
Parents: 186d195
Author: arp <ar...@apache.org>
Authored: Mon Sep 8 14:29:30 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../server/blockmanagement/BlockManager.java | 4 +-
.../hdfs/server/datanode/BPOfferService.java | 6 -
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 101 +++++++-----
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 157 ++++++++-----------
7 files changed, 130 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e12b01a..f6c0180 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -129,6 +129,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec";
public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60;
+ public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT = "dfs.datanode.ram.disk.low.watermark.percent";
+ public static final int DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT_DEFAULT = 10;
+ public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS = "dfs.datanode.ram.disk.low.watermark.replicas";
+ public static final int DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT = 3;
public static final String DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT =
"dfs.namenode.path.based.cache.block.map.allocation.percent";
public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 81b9bb5..e95f705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2119,8 +2119,8 @@ public class BlockManager {
// Add replica if appropriate. If the replica was previously corrupt
// but now okay, it might need to be updated.
if (reportedState == ReplicaState.FINALIZED
- && (!storedBlock.findDatanode(dn)
- || corruptReplicas.isReplicaCorrupt(storedBlock, dn))) {
+ && (storedBlock.findStorageInfo(storageInfo) == -1 ||
+ corruptReplicas.isReplicaCorrupt(storedBlock, dn))) {
toAdd.add(storedBlock);
}
return storedBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 2eab87f..1455a8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -230,7 +230,6 @@ class BPOfferService {
void notifyNamenodeReceivedBlock(
ExtendedBlock block, String delHint, String storageUuid) {
checkBlock(block);
- checkDelHint(delHint);
ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
block.getLocalBlock(),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
@@ -249,11 +248,6 @@ class BPOfferService {
block.getBlockPoolId(), getBlockPoolId());
}
- private void checkDelHint(String delHint) {
- Preconditions.checkArgument(delHint != null,
- "delHint is null");
- }
-
void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) {
checkBlock(block);
ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index a4db56a..bf09899 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -897,7 +897,7 @@ public class DataNode extends ReconfigurableBase
}
// calls specific to BP
- protected void notifyNamenodeReceivedBlock(
+ public void notifyNamenodeReceivedBlock(
ExtendedBlock block, String delHint, String storageUuid) {
BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId());
if(bpos != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 19260db..2b16a65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -47,7 +47,6 @@ import javax.management.StandardMBean;
import com.google.common.collect.Lists;
import com.google.common.base.Preconditions;
-import com.google.common.collect.TreeMultimap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -282,9 +281,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
cacheManager = new FsDatasetCache(this);
// Start the lazy writer once we have built the replica maps.
- lazyWriter = new Daemon(new LazyWriter(
- conf.getInt(DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
- DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC)));
+ lazyWriter = new Daemon(new LazyWriter(conf));
lazyWriter.start();
registerMBean(datanode.getDatanodeUuid());
}
@@ -2294,16 +2291,23 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
private volatile boolean shouldRun = true;
final int checkpointerInterval;
final long estimateBlockSize;
+ final int lowWatermarkFreeSpacePercentage;
+ final int lowWatermarkFreeSpaceReplicas;
- public static final int LOW_WATERMARK_FREE_SPACE_PERCENT = 10;
- public static final int LOW_WATERMARK_FREE_SPACE_REPLICAS = 3;
-
- public LazyWriter(final int checkpointerInterval) {
- this.checkpointerInterval = checkpointerInterval;
+ public LazyWriter(Configuration conf) {
+ this.checkpointerInterval = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
+ DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC);
this.estimateBlockSize = conf.getLongBytes(
DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+ this.lowWatermarkFreeSpacePercentage = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT,
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT_DEFAULT);
+ this.lowWatermarkFreeSpaceReplicas = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS,
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT);
}
private void moveReplicaToNewVolume(String bpid, long blockId)
@@ -2390,49 +2394,63 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
int percentFree = (int) (free * 100 / capacity);
- return percentFree < LOW_WATERMARK_FREE_SPACE_PERCENT ||
- free < (estimateBlockSize * LOW_WATERMARK_FREE_SPACE_REPLICAS);
+ return percentFree < lowWatermarkFreeSpacePercentage ||
+ free < (estimateBlockSize * lowWatermarkFreeSpaceReplicas);
}
/**
* Attempt to evict one or more transient block replicas we have at least
* spaceNeeded bytes free.
*/
- private synchronized void evictBlocks() throws IOException {
+ private void evictBlocks() throws IOException {
int iterations = 0;
- LazyWriteReplicaTracker.ReplicaState replicaState =
- lazyWriteReplicaTracker.getNextCandidateForEviction();
-
- while (replicaState != null &&
- iterations++ < MAX_BLOCK_EVICTIONS_PER_ITERATION &
+ while (iterations++ < MAX_BLOCK_EVICTIONS_PER_ITERATION &&
transientFreeSpaceBelowThreshold()) {
+ LazyWriteReplicaTracker.ReplicaState replicaState =
+ lazyWriteReplicaTracker.getNextCandidateForEviction();
+
if (LOG.isDebugEnabled()) {
- LOG.info("Evicting block " + replicaState);
+ LOG.debug("Evicting block " + replicaState);
}
- ReplicaInfo replicaInfo = getReplicaInfo(replicaState.bpid, replicaState.blockId);
- Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
- File blockFile = replicaInfo.getBlockFile();
- File metaFile = replicaInfo.getMetaFile();
- long blockFileUsed = blockFile.length();
- long metaFileUsed = metaFile.length();
- lazyWriteReplicaTracker.discardReplica(replicaState, false);
-
- // Move the replica from lazyPersist/ to finalized/ on target volume
- BlockPoolSlice bpSlice =
- replicaState.lazyPersistVolume.getBlockPoolSlice(replicaState.bpid);
- File newBlockFile = bpSlice.activateSavedReplica(
- replicaInfo, replicaState.savedBlockFile);
-
- ReplicaInfo newReplicaInfo =
- new FinalizedReplica(replicaInfo.getBlockId(),
- replicaInfo.getBytesOnDisk(),
- replicaInfo.getGenerationStamp(),
- replicaState.lazyPersistVolume,
- newBlockFile.getParentFile());
-
- // Update the volumeMap entry. This removes the old entry.
- volumeMap.add(replicaState.bpid, newReplicaInfo);
+
+ ReplicaInfo replicaInfo, newReplicaInfo;
+ File blockFile, metaFile;
+ long blockFileUsed, metaFileUsed;
+
+ synchronized (FsDatasetImpl.this) {
+ replicaInfo = getReplicaInfo(replicaState.bpid, replicaState.blockId);
+ Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
+ blockFile = replicaInfo.getBlockFile();
+ metaFile = replicaInfo.getMetaFile();
+ blockFileUsed = blockFile.length();
+ metaFileUsed = metaFile.length();
+ lazyWriteReplicaTracker.discardReplica(replicaState, false);
+
+ // Move the replica from lazyPersist/ to finalized/ on target volume
+ BlockPoolSlice bpSlice =
+ replicaState.lazyPersistVolume.getBlockPoolSlice(replicaState.bpid);
+ File newBlockFile = bpSlice.activateSavedReplica(
+ replicaInfo, replicaState.savedBlockFile);
+
+ newReplicaInfo =
+ new FinalizedReplica(replicaInfo.getBlockId(),
+ replicaInfo.getBytesOnDisk(),
+ replicaInfo.getGenerationStamp(),
+ replicaState.lazyPersistVolume,
+ newBlockFile.getParentFile());
+
+ // Update the volumeMap entry.
+ volumeMap.add(replicaState.bpid, newReplicaInfo);
+ }
+
+ // Before deleting the files from transient storage we must notify the
+ // NN that the files are on the new storage. Else a blockReport from
+ // the transient storage might cause the NN to think the blocks are lost.
+ ExtendedBlock extendedBlock =
+ new ExtendedBlock(replicaState.bpid, newReplicaInfo);
+ datanode.notifyNamenodeReceivedBlock(
+ extendedBlock, null, newReplicaInfo.getStorageUuid());
// Remove the old replicas from transient storage.
if (blockFile.delete() || !blockFile.exists()) {
@@ -2444,7 +2462,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// If deletion failed then the directory scanner will cleanup the blocks
// eventually.
- replicaState = lazyWriteReplicaTracker.getNextCandidateForEviction();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 338744d..7034715 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1343,6 +1343,7 @@ public class MiniDFSCluster {
}
int curDatanodesNum = dataNodes.size();
+ final int curDatanodesNumSaved = curDatanodesNum;
// for mincluster's the default initialDelay for BRs is 0
if (conf.get(DFS_BLOCKREPORT_INITIAL_DELAY_KEY) == null) {
conf.setLong(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, 0);
@@ -1481,7 +1482,7 @@ public class MiniDFSCluster {
waitActive();
if (storageCapacities != null) {
- for (int i = curDatanodesNum; i < curDatanodesNum+numDataNodes; ++i) {
+ for (int i = curDatanodesNumSaved; i < curDatanodesNumSaved+numDataNodes; ++i) {
final int index = i - curDatanodesNum;
List<? extends FsVolumeSpi> volumes = dns[index].getFSDataset().getVolumes();
assert storageCapacities[index].length == storagesPerDatanode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcec6582/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 7dfba50..fcc4798 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -71,13 +70,14 @@ public class TestLazyPersistFiles {
private static final int THREADPOOL_SIZE = 10;
- private static short REPL_FACTOR = 1;
+ private static final short REPL_FACTOR = 1;
private static final int BLOCK_SIZE = 10485760; // 10 MB
private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
private static final long HEARTBEAT_INTERVAL_SEC = 1;
private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
private static final int LAZY_WRITER_INTERVAL_SEC = 1;
private static final int BUFFER_LENGTH = 4096;
+ private static final int EVICTION_LOW_WATERMARK = 1;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
@@ -101,7 +101,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testFlagNotSetByDefault() throws IOException {
- startUpCluster(REPL_FACTOR, null, -1);
+ startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -113,7 +113,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testFlagPropagation() throws IOException {
- startUpCluster(REPL_FACTOR, null, -1);
+ startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -125,7 +125,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testFlagPersistenceInEditLog() throws IOException {
- startUpCluster(REPL_FACTOR, null, -1);
+ startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -139,10 +139,9 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testFlagPersistenceInFsImage() throws IOException {
- startUpCluster(REPL_FACTOR, null, -1);
+ startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
- FSDataOutputStream fos = null;
makeTestFile(path, 0, true);
// checkpoint
@@ -158,7 +157,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testPlacementOnRamDisk() throws IOException {
- startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK}, -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -168,8 +167,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testPlacementOnSizeLimitedRamDisk() throws IOException {
- startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK },
- 3 * BLOCK_SIZE -1); // 2 replicas + delta
+ startUpCluster(true, 3);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
@@ -188,7 +186,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testFallbackToDisk() throws IOException {
- startUpCluster(REPL_FACTOR, null, -1);
+ startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -202,7 +200,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testFallbackToDiskFull() throws IOException {
- startUpCluster(REPL_FACTOR, null, BLOCK_SIZE - 1);
+ startUpCluster(false, 0);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -213,15 +211,13 @@ public class TestLazyPersistFiles {
/**
* File partially fit in RamDisk after eviction.
* RamDisk can fit 2 blocks. Write a file with 5 blocks.
- * Expect 2 blocks are on RamDisk whereas other 3 on disk.
+ * Expect 2 or less blocks are on RamDisk and 3 or more on disk.
* @throws IOException
*/
@Test (timeout=300000)
public void testFallbackToDiskPartial()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR,
- new StorageType[] { RAM_DISK, DEFAULT },
- BLOCK_SIZE * 3 - 1);
+ startUpCluster(true, 2);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -241,12 +237,15 @@ public class TestLazyPersistFiles {
for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
if (locatedBlock.getStorageTypes()[0] == RAM_DISK) {
numBlocksOnRamDisk++;
- }else if (locatedBlock.getStorageTypes()[0] == DEFAULT) {
+ } else if (locatedBlock.getStorageTypes()[0] == DEFAULT) {
numBlocksOnDisk++;
}
}
- assertThat(numBlocksOnRamDisk, is(2));
- assertThat(numBlocksOnDisk, is(3));
+
+ // Since eviction is asynchronous, depending on the timing of eviction
+ // wrt writes, we may get 2 or less blocks on RAM disk.
+ assert(numBlocksOnRamDisk <= 2);
+ assert(numBlocksOnDisk >= 3);
}
/**
@@ -257,7 +256,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testRamDiskNotChosenByDefault() throws IOException {
- startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, RAM_DISK}, -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -275,7 +274,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testAppendIsDenied() throws IOException {
- startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -297,17 +296,12 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testLazyPersistFilesAreDiscarded()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR,
- new StorageType[] { RAM_DISK, DEFAULT },
- (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ startUpCluster(true, 2);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
- Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
makeTestFile(path1, BLOCK_SIZE, true);
- makeTestFile(path2, BLOCK_SIZE, false);
ensureFileReplicasOnStorageType(path1, RAM_DISK);
- ensureFileReplicasOnStorageType(path2, DEFAULT);
// Stop the DataNode and sleep for the time it takes the NN to
// detect the DN as being dead.
@@ -315,30 +309,28 @@ public class TestLazyPersistFiles {
Thread.sleep(30000L);
assertThat(cluster.getNamesystem().getNumDeadDataNodes(), is(1));
- // Next, wait for the replication monitor to mark the file as
- // corrupt, plus some delta.
+ // Next, wait for the replication monitor to mark the file as corrupt
Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000);
- // Wait for the LazyPersistFileScrubber to run, plus some delta.
+ // Wait for the LazyPersistFileScrubber to run
Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000);
// Ensure that path1 does not exist anymore, whereas path2 does.
assert(!fs.exists(path1));
- assert(fs.exists(path2));
- // We should have only one block that needs replication i.e. the one
+ // We should have zero blocks that needs replication i.e. the one
// belonging to path2.
assertThat(cluster.getNameNode()
.getNamesystem()
.getBlockManager()
.getUnderReplicatedBlocksCount(),
- is(1L));
+ is(0L));
}
@Test (timeout=300000)
public void testLazyPersistBlocksAreSaved()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT }, -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -386,16 +378,12 @@ public class TestLazyPersistFiles {
/**
* RamDisk eviction after lazy persist to disk.
- * Evicted blocks are still readable with on-disk replicas.
* @throws IOException
* @throws InterruptedException
*/
- @Test (timeout=300000)
- public void testRamDiskEviction()
- throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR,
- new StorageType[] { RAM_DISK, DEFAULT },
- (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ @Test (timeout=300000)
+ public void testRamDiskEviction() throws IOException, InterruptedException {
+ startUpCluster(true, 1 + EVICTION_LOW_WATERMARK);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
@@ -405,16 +393,16 @@ public class TestLazyPersistFiles {
ensureFileReplicasOnStorageType(path1, RAM_DISK);
// Sleep for a short time to allow the lazy writer thread to do its job.
- // However the block replica should not be evicted from RAM_DISK yet.
Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
ensureFileReplicasOnStorageType(path1, RAM_DISK);
// Create another file with a replica on RAM_DISK.
makeTestFile(path2, BLOCK_SIZE, true);
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
triggerBlockReport();
- // Make sure that the second file's block replica is on RAM_DISK, whereas
- // the original file's block replica is now on disk.
+ // Ensure the first file was evicted to disk, the second is still on
+ // RAM_DISK.
ensureFileReplicasOnStorageType(path2, RAM_DISK);
ensureFileReplicasOnStorageType(path1, DEFAULT);
}
@@ -428,9 +416,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testRamDiskEvictionBeforePersist()
throws IOException, InterruptedException {
- // 1 replica + delta, lazy persist interval every 50 minutes
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
- (2 * BLOCK_SIZE - 1));
+ startUpCluster(true, 1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
@@ -463,8 +449,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testRamDiskEvictionLRU()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
- (4 * BLOCK_SIZE -1)); // 3 replica + delta.
+ startUpCluster(true, 3);
final String METHOD_NAME = GenericTestUtils.getMethodName();
final int NUM_PATHS = 6;
Path paths[] = new Path[NUM_PATHS];
@@ -501,8 +486,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testDeleteBeforePersist()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
- -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
stopLazyWriter(cluster.getDataNodes().get(0));
@@ -527,7 +511,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testDeleteAfterPersist()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT }, -1);
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -554,8 +538,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testDfsUsageCreateDelete()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
- 5 * BLOCK_SIZE - 1); // 4 replica + delta
+ startUpCluster(true, 4);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -586,8 +569,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testConcurrentRead()
throws Exception {
- startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK },
- 3 * BLOCK_SIZE -1); // 2 replicas + delta
+ startUpCluster(true, 2);
final String METHOD_NAME = GenericTestUtils.getMethodName();
final Path path1 = new Path("/" + METHOD_NAME + ".dat");
@@ -638,8 +620,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testConcurrentWrites()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
- (10 * BLOCK_SIZE -1)); // 9 replica + delta.
+ startUpCluster(true, 9);
final String METHOD_NAME = GenericTestUtils.getMethodName();
final int SEED = 0xFADED;
final int NUM_WRITERS = 4;
@@ -659,8 +640,7 @@ public class TestLazyPersistFiles {
ExecutorService executor = Executors.newFixedThreadPool(THREADPOOL_SIZE);
for (int i = 0; i < NUM_WRITERS; i++) {
- Runnable writer = new WriterRunnable(cluster, i, paths[i], SEED, latch,
- testFailed);
+ Runnable writer = new WriterRunnable(i, paths[i], SEED, latch, testFailed);
executor.execute(writer);
}
@@ -677,9 +657,7 @@ public class TestLazyPersistFiles {
public void testDnRestartWithSavedReplicas()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR,
- new StorageType[] {RAM_DISK, DEFAULT },
- (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -703,9 +681,7 @@ public class TestLazyPersistFiles {
public void testDnRestartWithUnsavedReplicas()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR,
- new StorageType[] {RAM_DISK, DEFAULT },
- (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ startUpCluster(true, 1);
stopLazyWriter(cluster.getDataNodes().get(0));
final String METHOD_NAME = GenericTestUtils.getMethodName();
@@ -727,9 +703,8 @@ public class TestLazyPersistFiles {
* If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially
* capped. If ramDiskStorageLimit < 0 then it is ignored.
*/
- private void startUpCluster(final int numDataNodes,
- final StorageType[] storageTypes,
- final long ramDiskStorageLimit,
+ private void startUpCluster(boolean hasTransientStorage,
+ final int ramDiskReplicaCapacity,
final boolean useSCR)
throws IOException {
@@ -739,42 +714,36 @@ public class TestLazyPersistFiles {
LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
- HEARTBEAT_RECHECK_INTERVAL_MSEC);
+ HEARTBEAT_RECHECK_INTERVAL_MSEC);
conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
- LAZY_WRITER_INTERVAL_SEC);
+ LAZY_WRITER_INTERVAL_SEC);
+ conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS,
+ EVICTION_LOW_WATERMARK);
- conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY,useSCR);
+ conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, useSCR);
- REPL_FACTOR = 1; //Reset in case a test has modified the value
+ long[] capacities = null;
+ if (hasTransientStorage && ramDiskReplicaCapacity >= 0) {
+ // Convert replica count to byte count, add some delta for .meta and VERSION files.
+ long ramDiskStorageLimit = ((long) ramDiskReplicaCapacity * BLOCK_SIZE) + (BLOCK_SIZE - 1);
+ capacities = new long[] { ramDiskStorageLimit, -1 };
+ }
cluster = new MiniDFSCluster
.Builder(conf)
- .numDataNodes(numDataNodes)
- .storageTypes(storageTypes != null ? storageTypes : new StorageType[] { DEFAULT, DEFAULT })
+ .numDataNodes(REPL_FACTOR)
+ .storageCapacities(capacities)
+ .storageTypes(hasTransientStorage ? new StorageType[]{ RAM_DISK, DEFAULT } : null)
.build();
fs = cluster.getFileSystem();
client = fs.getClient();
-
- // Artificially cap the storage capacity of the RAM_DISK volume.
- if (ramDiskStorageLimit >= 0) {
- List<? extends FsVolumeSpi> volumes =
- cluster.getDataNodes().get(0).getFSDataset().getVolumes();
-
- for (FsVolumeSpi volume : volumes) {
- if (volume.getStorageType() == RAM_DISK) {
- ((FsTransientVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
- }
- }
- }
-
LOG.info("Cluster startup complete");
}
- private void startUpCluster(final int numDataNodes,
- final StorageType[] storageTypes,
- final long ramDiskStorageLimit)
+ private void startUpCluster(boolean hasTransientStorage,
+ final int ramDiskReplicaCapacity)
throws IOException {
- startUpCluster(numDataNodes, storageTypes, ramDiskStorageLimit, false);
+ startUpCluster(hasTransientStorage, ramDiskReplicaCapacity, false);
}
private void makeTestFile(Path path, long length, final boolean isLazyPersist)
@@ -908,17 +877,15 @@ public class TestLazyPersistFiles {
class WriterRunnable implements Runnable {
private final int id;
- private final MiniDFSCluster cluster;
private final Path paths[];
private final int seed;
private CountDownLatch latch;
private AtomicBoolean bFail;
- public WriterRunnable(MiniDFSCluster cluster, int threadIndex, Path[] paths,
+ public WriterRunnable(int threadIndex, Path[] paths,
int seed, CountDownLatch latch,
AtomicBoolean bFail) {
id = threadIndex;
- this.cluster = cluster;
this.paths = paths;
this.seed = seed;
this.latch = latch;
[31/34] git commit: HDFS-7090. Use unbuffered writes when persisting
in-memory replicas. Contributed by Xiaoyu Yao.
Posted by ji...@apache.org.
HDFS-7090. Use unbuffered writes when persisting in-memory replicas. Contributed by Xiaoyu Yao.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e98379b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e98379b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e98379b
Branch: refs/heads/branch-2.6
Commit: 2e98379bcb47235afb97c09bcb90d539cbd534fc
Parents: 1b6d115
Author: cnauroth <cn...@apache.org>
Authored: Mon Oct 13 10:50:25 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:53 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/io/nativeio/Errno.java | 1 +
.../org/apache/hadoop/io/nativeio/NativeIO.java | 27 ++++++
.../org/apache/hadoop/io/nativeio/NativeIO.c | 67 +++++++++++++++
.../org/apache/hadoop/io/nativeio/errno_enum.c | 1 +
.../apache/hadoop/io/nativeio/TestNativeIO.java | 34 ++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/common/Storage.java | 88 ++++++++++++++++++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 4 +-
8 files changed, 223 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java
index f823978..f6377c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java
@@ -58,6 +58,7 @@ public enum Errno {
ELOOP,
ENAMETOOLONG,
ENOTEMPTY,
+ EOVERFLOW,
UNKNOWN;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 53d31d6..43f1cb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -29,6 +29,7 @@ import java.nio.MappedByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -846,4 +847,30 @@ public class NativeIO {
private static native void link0(String src, String dst)
throws NativeIOException;
+
+ /**
+ * Unbuffered file copy from src to dst without tainting OS buffer cache
+ * In Linux, it uses sendfile() which uses O_DIRECT flag internally
+ * In Windows, it uses CopyFileEx with COPY_FILE_NO_BUFFERING flag
+ *
+ * Note: This does not support FreeBSD/OSX which have a different sendfile()
+ * semantic. Also, this simple native wrapper does minimal parameter checking
+ * It is recommended to use wrapper function like
+ * the Storage#nativeCopyFileUnbuffered() function in hadoop-hdfs.
+ *
+ *
+ * @param src The source path
+ * @param dst The destination path
+ * @throws IOException
+ */
+ public static void copyFileUnbuffered(File src, File dst) throws IOException {
+ if ((nativeLoaded) && (Shell.WINDOWS || Shell.LINUX)) {
+ copyFileUnbuffered0(src.getAbsolutePath(), dst.getAbsolutePath());
+ } else {
+ FileUtils.copyFile(src, dst);
+ }
+ }
+
+ private static native void copyFileUnbuffered0(String src, String dst)
+ throws NativeIOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index b98aa0c..f19d6be 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -35,6 +35,9 @@
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/syscall.h>
+#if !(defined(__FreeBSD__) || defined(__MACH__))
+#include <sys/sendfile.h>
+#endif
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
@@ -1142,6 +1145,70 @@ JNIEnv *env, jclass clazz)
#endif
}
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_copyFileUnbuffered0(
+JNIEnv *env, jclass clazz, jstring jsrc, jstring jdst)
+{
+#ifdef UNIX
+#if (defined(__FreeBSD__) || defined(__MACH__))
+ THROW(env, "java/io/IOException",
+ "The function copyFileUnbuffered() is not supported on FreeBSD or Mac OS");
+ return;
+#else
+ const char *src = NULL, *dst = NULL;
+ int srcFd = -1;
+ int dstFd = -1;
+ struct stat s;
+ off_t offset = 0;
+
+ src = (*env)->GetStringUTFChars(env, jsrc, NULL);
+ if (!src) goto cleanup; // exception was thrown
+ dst = (*env)->GetStringUTFChars(env, jdst, NULL);
+ if (!dst) goto cleanup; // exception was thrown
+
+ srcFd = open(src, O_RDONLY);
+ if (srcFd == -1) {
+ throw_ioe(env, errno);
+ goto cleanup;
+ }
+ if (fstat(srcFd, &s) == -1){
+ throw_ioe(env, errno);
+ goto cleanup;
+ }
+ dstFd = open(dst, O_WRONLY | O_CREAT, s.st_mode);
+ if (dstFd == -1) {
+ throw_ioe(env, errno);
+ goto cleanup;
+ }
+ if (sendfile(dstFd, srcFd, &offset, s.st_size) == -1) {
+ throw_ioe(env, errno);
+ }
+
+cleanup:
+ if (src) (*env)->ReleaseStringUTFChars(env, jsrc, src);
+ if (dst) (*env)->ReleaseStringUTFChars(env, jdst, dst);
+ if (srcFd != -1) close(srcFd);
+ if (dstFd != -1) close(dstFd);
+#endif
+#endif
+
+#ifdef WINDOWS
+ LPCWSTR src = NULL, dst = NULL;
+
+ src = (LPCWSTR) (*env)->GetStringChars(env, jsrc, NULL);
+ if (!src) goto cleanup; // exception was thrown
+ dst = (LPCWSTR) (*env)->GetStringChars(env, jdst, NULL);
+ if (!dst) goto cleanup; // exception was thrown
+ if (!CopyFileEx(src, dst, NULL, NULL, NULL, COPY_FILE_NO_BUFFERING)) {
+ throw_ioe(env, GetLastError());
+ }
+
+cleanup:
+ if (src) (*env)->ReleaseStringChars(env, jsrc, src);
+ if (dst) (*env)->ReleaseStringChars(env, jdst, dst);
+#endif
+}
+
/**
* vim: sw=2: ts=2: et:
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c
index 4d07c31..08cc305 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c
@@ -66,6 +66,7 @@ static errno_mapping_t ERRNO_MAPPINGS[] = {
MAPPING(ELOOP),
MAPPING(ENAMETOOLONG),
MAPPING(ENOTEMPTY),
+ MAPPING(EOVERFLOW),
{-1, NULL}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
index 6c3f003..5425c49 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
@@ -24,14 +24,18 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
+import java.util.Random;
import java.util.concurrent.atomic.AtomicReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -622,4 +626,34 @@ public class TestNativeIO {
assumeTrue(NativeIO.isAvailable());
NativeIO.getMemlockLimit();
}
+
+ @Test (timeout = 30000)
+ public void testCopyFileUnbuffered() throws Exception {
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ File srcFile = new File(TEST_DIR, METHOD_NAME + ".src.dat");
+ File dstFile = new File(TEST_DIR, METHOD_NAME + ".dst.dat");
+ final int fileSize = 0x8FFFFFF; // 128 MB
+ final int SEED = 0xBEEF;
+ final int batchSize = 4096;
+ final int numBatches = fileSize / batchSize;
+ Random rb = new Random(SEED);
+ FileChannel channel = null;
+ RandomAccessFile raSrcFile = null;
+ try {
+ raSrcFile = new RandomAccessFile(srcFile, "rw");
+ channel = raSrcFile.getChannel();
+ byte bytesToWrite[] = new byte[batchSize];
+ MappedByteBuffer mapBuf;
+ mapBuf = channel.map(MapMode.READ_WRITE, 0, fileSize);
+ for (int i = 0; i < numBatches; i++) {
+ rb.nextBytes(bytesToWrite);
+ mapBuf.put(bytesToWrite);
+ }
+ NativeIO.copyFileUnbuffered(srcFile, dstFile);
+ }finally {
+ IOUtils.cleanup(LOG, channel);
+ IOUtils.cleanup(LOG, raSrcFile);
+ FileUtils.deleteQuietly(TEST_DIR);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index afb8d35..55dc567 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -665,6 +665,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7112. LazyWriter should use either async IO or one thread per physical
disk. (Xiaoyu Yao via cnauroth)
+ HDFS-7090. Use unbuffered writes when persisting in-memory replicas.
+ (Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 0661026..4320e22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.common;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.management.ManagementFactory;
@@ -997,6 +998,93 @@ public abstract class Storage extends StorageInfo {
}
/**
+ * Copies a file (usually large) to a new location using native unbuffered IO.
+ * <p>
+ * This method copies the contents of the specified source file
+ * to the specified destination file using OS specific unbuffered IO.
+ * The goal is to avoid churning the file system buffer cache when copying
+ * large files. TheFileUtils#copyLarge function from apache-commons-io library
+ * can be used to achieve this with an internal memory buffer but is less
+ * efficient than the native unbuffered APIs such as sendfile() in Linux and
+ * CopyFileEx() in Windows wrapped in {@link NativeIO#copyFileUnbuffered}.
+ *
+ * The directory holding the destination file is created if it does not exist.
+ * If the destination file exists, then this method will delete it first.
+ * <p>
+ * <strong>Note:</strong> Setting <code>preserveFileDate</code> to
+ * {@code true} tries to preserve the file's last modified
+ * date/times using {@link File#setLastModified(long)}, however it is
+ * not guaranteed that the operation will succeed.
+ * If the modification operation fails, no indication is provided.
+ *
+ * @param srcFile an existing file to copy, must not be {@code null}
+ * @param destFile the new file, must not be {@code null}
+ * @param preserveFileDate true if the file date of the copy
+ * should be the same as the original
+ *
+ * @throws NullPointerException if source or destination is {@code null}
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void nativeCopyFileUnbuffered(File srcFile, File destFile,
+ boolean preserveFileDate) throws IOException {
+ if (srcFile == null) {
+ throw new NullPointerException("Source must not be null");
+ }
+ if (destFile == null) {
+ throw new NullPointerException("Destination must not be null");
+ }
+ if (srcFile.exists() == false) {
+ throw new FileNotFoundException("Source '" + srcFile + "' does not exist");
+ }
+ if (srcFile.isDirectory()) {
+ throw new IOException("Source '" + srcFile + "' exists but is a directory");
+ }
+ if (srcFile.getCanonicalPath().equals(destFile.getCanonicalPath())) {
+ throw new IOException("Source '" + srcFile + "' and destination '" +
+ destFile + "' are the same");
+ }
+ File parentFile = destFile.getParentFile();
+ if (parentFile != null) {
+ if (!parentFile.mkdirs() && !parentFile.isDirectory()) {
+ throw new IOException("Destination '" + parentFile
+ + "' directory cannot be created");
+ }
+ }
+ if (destFile.exists()) {
+ if (FileUtil.canWrite(destFile) == false) {
+ throw new IOException("Destination '" + destFile
+ + "' exists but is read-only");
+ } else {
+ if (destFile.delete() == false) {
+ throw new IOException("Destination '" + destFile
+ + "' exists but cannot be deleted");
+ }
+ }
+ }
+ try {
+ NativeIO.copyFileUnbuffered(srcFile, destFile);
+ } catch (NativeIOException e) {
+ throw new IOException("Failed to copy " + srcFile.getCanonicalPath()
+ + " to " + destFile.getCanonicalPath()
+ + " due to failure in NativeIO#copyFileUnbuffered(). "
+ + e.toString());
+ }
+ if (srcFile.length() != destFile.length()) {
+ throw new IOException("Failed to copy full contents from '" + srcFile
+ + "' to '" + destFile + "'");
+ }
+ if (preserveFileDate) {
+ if (destFile.setLastModified(srcFile.lastModified()) == false) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to preserve last modified date from'" + srcFile
+ + "' to '" + destFile + "'");
+ }
+ }
+ }
+ }
+
+ /**
* Recursively delete all the content of the directory first and then
* the directory itself from the local filesystem.
* @param dir The directory to delete
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e98379b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 07e19cf..1709066 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -724,12 +724,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final File dstFile = new File(destDir, srcFile.getName());
final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp);
try {
- FileUtils.copyFile(srcMeta, dstMeta);
+ Storage.nativeCopyFileUnbuffered(srcMeta, dstMeta, true);
} catch (IOException e) {
throw new IOException("Failed to copy " + srcMeta + " to " + dstMeta, e);
}
try {
- FileUtils.copyFile(srcFile, dstFile);
+ Storage.nativeCopyFileUnbuffered(srcFile, dstFile, true);
} catch (IOException e) {
throw new IOException("Failed to copy " + srcFile + " to " + dstFile, e);
}
[12/34] git commit: HDFS-6960. Bugfix in LazyWriter,
fix test case and some refactoring. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6960. Bugfix in LazyWriter, fix test case and some refactoring. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d973656e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d973656e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d973656e
Branch: refs/heads/branch-2.6
Commit: d973656e5d20351a65ab78871c4c77e14d150bd6
Parents: 2079d4b
Author: arp <ar...@apache.org>
Authored: Thu Aug 28 23:05:32 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/BlockPoolSlice.java | 11 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 129 +++++++++----------
.../datanode/fsdataset/impl/FsVolumeImpl.java | 16 ---
.../fsdataset/impl/LazyWriteReplicaTracker.java | 76 +++++++++--
.../fsdataset/impl/TestLazyPersistFiles.java | 10 +-
5 files changed, 138 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d973656e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 179f3ea..63cac55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -277,10 +277,13 @@ class BlockPoolSlice {
return blockFile;
}
- File lazyPersistReplica(Block b, File f) throws IOException {
- File blockFile = FsDatasetImpl.copyBlockFiles(b, f, lazypersistDir);
- File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
- dfsUsage.incDfsUsed(b.getNumBytes() + metaFile.length());
+ File lazyPersistReplica(ReplicaInfo replicaInfo) throws IOException {
+ if (!lazypersistDir.exists() && !lazypersistDir.mkdirs()) {
+ FsDatasetImpl.LOG.warn("Failed to create " + lazypersistDir);
+ }
+ File metaFile = FsDatasetImpl.copyBlockFiles(replicaInfo, lazypersistDir);
+ File blockFile = Block.metaToBlockFile(metaFile);
+ dfsUsage.incDfsUsed(replicaInfo.getNumBytes() + metaFile.length());
return blockFile;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d973656e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 1abc6f9..5c822e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.Replica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException;
@@ -704,28 +705,33 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
return dstfile;
}
- static File copyBlockFiles(Block b, File srcfile, File destdir)
+ /**
+ * Copy the block and meta files for the given block from the given
+ * @return the new meta file.
+ * @throws IOException
+ */
+ static File copyBlockFiles(ReplicaInfo replicaInfo, File destRoot)
throws IOException {
- final File dstfile = new File(destdir, b.getBlockName());
- final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp());
- final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp());
+ final File destDir = DatanodeUtil.idToBlockDir(destRoot, replicaInfo.getBlockId());
+ final File dstFile = new File(destDir, replicaInfo.getBlockName());
+ final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, replicaInfo.getGenerationStamp());
+ final File srcMeta = replicaInfo.getMetaFile();
+ final File srcFile = replicaInfo.getBlockFile();
try {
- FileUtils.copyFile(srcmeta, dstmeta);
+ FileUtils.copyFile(srcMeta, dstMeta);
} catch (IOException e) {
- throw new IOException("Failed to copy meta file for " + b
- + " from " + srcmeta + " to " + dstmeta, e);
+ throw new IOException("Failed to copy " + srcMeta + " to " + dstMeta, e);
}
try {
- FileUtils.copyFile(srcfile, dstfile);
+ FileUtils.copyFile(srcFile, dstFile);
} catch (IOException e) {
- throw new IOException("Failed to copy block file for " + b
- + " from " + srcfile + " to " + dstfile.getAbsolutePath(), e);
+ throw new IOException("Failed to copy " + srcFile + " to " + dstFile, e);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("addBlock: Moved " + srcmeta + " to " + dstmeta
- + " and " + srcfile + " to " + dstfile);
+ LOG.debug("addBlock: Moved " + srcMeta + " to " + dstMeta);
+ LOG.debug("addBlock: Moved " + srcFile + " to " + dstFile);
}
- return dstfile;
+ return dstMeta;
}
static private void truncateBlock(File blockFile, File metaFile,
@@ -1322,10 +1328,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (v.isTransientStorage()) {
lazyWriteReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
-
- // Schedule a checkpoint.
- ((LazyWriter) lazyWriter.getRunnable())
- .addReplicaToLazyWriteQueue(bpid, replicaInfo.getBlockId());
}
}
volumeMap.add(bpid, newReplicaInfo);
@@ -2371,32 +2373,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
nbytes, flags);
}
- private static class BlockIdPair {
- final String bpid;
- final long blockId;
-
- BlockIdPair(final String bpid, final long blockId) {
- this.bpid = bpid;
- this.blockId = blockId;
- }
- }
-
- private class LazyWriter implements Runnable {
+ class LazyWriter implements Runnable {
private volatile boolean shouldRun = true;
final int checkpointerInterval;
- final private Queue<BlockIdPair> blocksPendingCheckpoint;
-
public LazyWriter(final int checkpointerInterval) {
this.checkpointerInterval = checkpointerInterval;
- blocksPendingCheckpoint = new LinkedList<BlockIdPair>();
- }
-
- // Schedule a replica for writing to persistent storage.
- public synchronized void addReplicaToLazyWriteQueue(
- String bpid, long blockId) {
- LOG.info("Block with blockId=" + blockId + "; bpid=" + bpid + " added to lazy writer queue");
- blocksPendingCheckpoint.add(new BlockIdPair(bpid, blockId));
}
private void moveReplicaToNewVolume(String bpid, long blockId)
@@ -2404,76 +2386,85 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.info("LazyWriter invoked to save blockId=" + blockId + "; bpid=" + bpid);
- FsVolumeImpl targetVolume = null;
- Block block = null;
- File blockFile = null;
+ FsVolumeImpl targetVolume;
+ ReplicaInfo replicaInfo;
synchronized (this) {
- block = getStoredBlock(bpid, blockId);
- blockFile = getFile(bpid, blockId);
+ replicaInfo = volumeMap.get(bpid, blockId);
- if (block == null) {
- // The block was deleted before it could be checkpointed.
+ if (replicaInfo == null || !replicaInfo.getVolume().isTransientStorage()) {
+ // The block was either deleted before it could be checkpointed or
+ // it is already on persistent storage. This can occur if a second
+ // replica on persistent storage was found after the lazy write was
+ // scheduled.
return;
}
// Pick a target volume for the block.
targetVolume = volumes.getNextVolume(
- StorageType.DEFAULT, block.getNumBytes());
+ StorageType.DEFAULT, replicaInfo.getNumBytes());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
}
- LOG.info("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
File savedBlockFile = targetVolume.getBlockPoolSlice(bpid)
- .lazyPersistReplica(block, blockFile);
+ .lazyPersistReplica(replicaInfo);
lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedBlockFile);
- LOG.info("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
- " to file " + savedBlockFile);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
+ " to file " + savedBlockFile);
+ }
}
/**
* Checkpoint a pending replica to persistent storage now.
+ * If we fail then move the replica to the end of the queue.
* @return true if there is more work to be done, false otherwise.
*/
private boolean saveNextReplica() {
- BlockIdPair blockIdPair = null;
- int moreWorkThreshold = 0;
+ LazyWriteReplicaTracker.ReplicaState replicaState = null;
+ boolean succeeded = false;
try {
synchronized (this) {
- // Dequeue the next replica waiting to be checkpointed.
- blockIdPair = blocksPendingCheckpoint.poll();
- if (blockIdPair == null) {
- LOG.info("LazyWriter has no blocks to persist. " +
- "Thread going to sleep.");
+ replicaState = lazyWriteReplicaTracker.dequeueNextReplicaToPersist();
+ if (replicaState == null) {
return false;
}
}
// Move the replica outside the lock.
- moveReplicaToNewVolume(blockIdPair.bpid, blockIdPair.blockId);
-
+ moveReplicaToNewVolume(replicaState.bpid, replicaState.blockId);
+ succeeded = true;
} catch(IOException ioe) {
- // If we failed, put the block on the queue and let a retry
- // interval elapse before we try again so we don't try to keep
- // checkpointing the same block in a tight loop.
- synchronized (this) {
- blocksPendingCheckpoint.add(blockIdPair);
- ++moreWorkThreshold;
+ LOG.warn("Exception saving replica " + replicaState, ioe);
+ } finally {
+ if (!succeeded && replicaState != null) {
+ lazyWriteReplicaTracker.reenqueueReplica(replicaState);
}
}
- synchronized (this) {
- return blocksPendingCheckpoint.size() > moreWorkThreshold;
- }
+ return succeeded;
}
@Override
public void run() {
+ int numSuccessiveFailures = 0;
+
while (fsRunning && shouldRun) {
try {
- if (!saveNextReplica()) {
+ numSuccessiveFailures = saveNextReplica() ? 0 : (numSuccessiveFailures + 1);
+
+ // Sleep if we have no more work to do or if it looks like we are not
+ // making any forward progress. This is to ensure that if all persist
+ // operations are failing we don't keep retrying them in a tight loop.
+ if (numSuccessiveFailures == lazyWriteReplicaTracker.numReplicasNotPersisted()) {
Thread.sleep(checkpointerInterval * 1000);
+ numSuccessiveFailures = 0;
}
} catch (InterruptedException e) {
LOG.info("LazyWriter was interrupted, exiting");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d973656e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 994344e..93b5895 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -307,22 +307,6 @@ public class FsVolumeImpl implements FsVolumeSpi {
getBlockPoolSlice(bpid).getVolumeMap(volumeMap);
}
- /**
- * Add replicas under the given directory to the volume map
- * @param volumeMap the replicas map
- * @param dir an input directory
- * @param isFinalized true if the directory has finalized replicas;
- * false if the directory has rbw replicas
- * @throws IOException
- */
- void addToReplicasMap(String bpid, ReplicaMap volumeMap,
- File dir, boolean isFinalized) throws IOException {
- BlockPoolSlice bp = getBlockPoolSlice(bpid);
- // TODO move this up
- // dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length());
- bp.addToReplicasMap(volumeMap, dir, isFinalized);
- }
-
@Override
public String toString() {
return currentDir.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d973656e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
index ae28f09..222b63a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
@@ -19,12 +19,11 @@
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
class LazyWriteReplicaTracker {
@@ -43,7 +42,7 @@ class LazyWriteReplicaTracker {
/**
* transient storage volume that holds the original replica.
*/
- final FsVolumeImpl transientVolume;
+ final FsVolumeSpi transientVolume;
/**
* Persistent volume that holds or will hold the saved replica.
@@ -51,7 +50,7 @@ class LazyWriteReplicaTracker {
FsVolumeImpl lazyPersistVolume;
File savedBlockFile;
- ReplicaState(final String bpid, final long blockId, FsVolumeImpl transientVolume) {
+ ReplicaState(final String bpid, final long blockId, FsVolumeSpi transientVolume) {
this.bpid = bpid;
this.blockId = blockId;
this.transientVolume = transientVolume;
@@ -61,6 +60,11 @@ class LazyWriteReplicaTracker {
}
@Override
+ public String toString() {
+ return "[Bpid=" + bpid + ";blockId=" + blockId + "]";
+ }
+
+ @Override
public int hashCode() {
return bpid.hashCode() ^ (int) blockId;
}
@@ -99,35 +103,43 @@ class LazyWriteReplicaTracker {
final Map<String, Map<Long, ReplicaState>> replicaMaps;
/**
+ * Queue of replicas that need to be written to disk.
+ */
+ final Queue<ReplicaState> replicasNotPersisted;
+
+ /**
* A map of blockId to persist complete time for transient blocks. This allows
* us to evict LRU blocks from transient storage. Protected by 'this'
* Object lock.
*/
- final Map<ReplicaState, Long> persistTimeMap;
+ final Map<ReplicaState, Long> replicasPersisted;
LazyWriteReplicaTracker(final FsDatasetImpl fsDataset) {
this.fsDataset = fsDataset;
replicaMaps = new HashMap<String, Map<Long, ReplicaState>>();
- persistTimeMap = new HashMap<ReplicaState, Long>();
+ replicasNotPersisted = new LinkedList<ReplicaState>();
+ replicasPersisted = new HashMap<ReplicaState, Long>();
}
TreeMultimap<Long, ReplicaState> getLruMap() {
// TODO: This can be made more efficient.
TreeMultimap<Long, ReplicaState> reversedMap = TreeMultimap.create();
- for (Map.Entry<ReplicaState, Long> entry : persistTimeMap.entrySet()) {
+ for (Map.Entry<ReplicaState, Long> entry : replicasPersisted.entrySet()) {
reversedMap.put(entry.getValue(), entry.getKey());
}
return reversedMap;
}
synchronized void addReplica(String bpid, long blockId,
- final FsVolumeImpl transientVolume) {
+ final FsVolumeSpi transientVolume) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
if (map == null) {
map = new HashMap<Long, ReplicaState>();
replicaMaps.put(bpid, map);
}
- map.put(blockId, new ReplicaState(bpid, blockId, transientVolume));
+ ReplicaState replicaState = new ReplicaState(bpid, blockId, transientVolume);
+ map.put(blockId, replicaState);
+ replicasNotPersisted.add(replicaState);
}
synchronized void recordStartLazyPersist(
@@ -149,12 +161,49 @@ class LazyWriteReplicaTracker {
}
replicaState.state = State.LAZY_PERSIST_COMPLETE;
replicaState.savedBlockFile = savedBlockFile;
- persistTimeMap.put(replicaState, System.currentTimeMillis() / 1000);
+
+ if (replicasNotPersisted.peek() == replicaState) {
+ // Common case.
+ replicasNotPersisted.remove();
+ } else {
+ // Should never occur in practice as lazy writer always persists
+ // the replica at the head of the queue before moving to the next
+ // one.
+ replicasNotPersisted.remove(replicaState);
+ }
+ replicasPersisted.put(replicaState, System.currentTimeMillis() / 1000);
+ }
+
+ synchronized ReplicaState dequeueNextReplicaToPersist() {
+ while (replicasNotPersisted.size() != 0) {
+ ReplicaState replicaState = replicasNotPersisted.remove();
+ Map<Long, ReplicaState> replicaMap = replicaMaps.get(replicaState.bpid);
+
+ if (replicaMap != null && replicaMap.get(replicaState.blockId) != null) {
+ return replicaState;
+ }
+
+ // The replica no longer exists, look for the next one.
+ }
+ return null;
+ }
+
+ synchronized void reenqueueReplica(final ReplicaState replicaState) {
+ replicasNotPersisted.add(replicaState);
+ }
+
+ synchronized int numReplicasNotPersisted() {
+ return replicasNotPersisted.size();
}
synchronized void discardReplica(
final String bpid, final long blockId, boolean force) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
+
+ if (map == null) {
+ return;
+ }
+
ReplicaState replicaState = map.get(blockId);
if (replicaState == null) {
@@ -172,6 +221,9 @@ class LazyWriteReplicaTracker {
}
map.remove(blockId);
- persistTimeMap.remove(replicaState);
+ replicasPersisted.remove(replicaState);
+
+ // Leave the replica in replicasNotPersisted if its present.
+ // dequeueNextReplicaToPersist will GC it eventually.
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d973656e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index ddd71b1..af0e8ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.test.GenericTestUtils;
@@ -61,6 +63,7 @@ public class TestLazyPersistFiles {
static {
((Log4JLogger) NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) FsDatasetImpl.LOG).getLogger().setLevel(Level.ALL);
}
private static short REPL_FACTOR = 1;
@@ -68,7 +71,7 @@ public class TestLazyPersistFiles {
private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
private static final long HEARTBEAT_INTERVAL_SEC = 1;
private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
- private static final int LAZY_WRITER_INTERVAL_SEC = 3;
+ private static final int LAZY_WRITER_INTERVAL_SEC = 1;
private static final int BUFFER_LENGTH = 4096;
private MiniDFSCluster cluster;
@@ -283,8 +286,9 @@ public class TestLazyPersistFiles {
File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
- File persistedBlockFile = new File(lazyPersistDir, "blk_" + lb.getBlock().getBlockId());
- if (persistedBlockFile.exists()) {
+ File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, lb.getBlock().getBlockId());
+ File blockFile = new File(targetDir, lb.getBlock().getBlockName());
+ if (blockFile.exists()) {
// Found a persisted copy for this block!
boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
assertThat(added, is(true));
[22/34] git commit: HDFS-7079. Few more unit test fixes for
HDFS-6581. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7079. Few more unit test fixes for HDFS-6581. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d168d9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d168d9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d168d9f
Branch: refs/heads/branch-2.6
Commit: 7d168d9ff9f1c2ba19a7fc698578c1c69b90de50
Parents: d3ccbd8
Author: arp <ar...@apache.org>
Authored: Tue Sep 16 18:19:57 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../hadoop/fs/shell/CommandWithDestination.java | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d168d9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 7152f02..65e52fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -473,19 +473,20 @@ abstract class CommandWithDestination extends FsCommand {
FSDataOutputStream create(PathData item, boolean lazyPersist)
throws IOException {
try {
- EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
if (lazyPersist) {
- createFlags.add(LAZY_PERSIST);
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE, LAZY_PERSIST);
+ return create(item.path,
+ FsPermission.getFileDefault().applyUMask(
+ FsPermission.getUMask(getConf())),
+ createFlags,
+ getConf().getInt("io.file.buffer.size", 4096),
+ lazyPersist ? 1 : getDefaultReplication(item.path),
+ getDefaultBlockSize(),
+ null,
+ null);
+ } else {
+ return create(item.path, true);
}
- return create(item.path,
- FsPermission.getFileDefault().applyUMask(
- FsPermission.getUMask(getConf())),
- createFlags,
- getConf().getInt("io.file.buffer.size", 4096),
- lazyPersist ? 1 : getDefaultReplication(item.path),
- getDefaultBlockSize(),
- null,
- null);
} finally { // might have been created but stream was interrupted
deleteOnExit(item.path);
}
[19/34] git commit: HDFS-7080. Fix finalize and upgrade unit test
failures. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7080. Fix finalize and upgrade unit test failures. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7062e8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7062e8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7062e8d
Branch: refs/heads/branch-2.6
Commit: b7062e8d3e1b2b775567899a33394a5d446abf9c
Parents: 7d168d9
Author: arp <ar...@apache.org>
Authored: Wed Sep 17 15:25:04 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++
.../server/datanode/fsdataset/impl/BlockPoolSlice.java | 10 ++++++++++
.../test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java | 3 +++
.../test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java | 3 +++
4 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7062e8d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f6c0180..f5bba86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -133,6 +133,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT_DEFAULT = 10;
public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS = "dfs.datanode.ram.disk.low.watermark.replicas";
public static final int DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT = 3;
+
+ // This setting is for testing/internal use only.
+ public static final String DFS_DATANODE_DUPLICATE_REPLICA_DELETION = "dfs.datanode.duplicate.replica.deletion";
+ public static final boolean DFS_DATANODE_DUPLICATE_REPLICA_DELETION_DEFAULT = true;
+
public static final String DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT =
"dfs.namenode.path.based.cache.block.map.allocation.percent";
public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7062e8d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index c370f3d..06d60b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -68,6 +68,7 @@ class BlockPoolSlice {
private static final String DU_CACHE_FILE = "dfsUsed";
private volatile boolean dfsUsedSaved = false;
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private final boolean deleteDuplicateReplicas;
// TODO:FEDERATION scalability issue - a thread per DU is needed
private final DU dfsUsage;
@@ -94,6 +95,10 @@ class BlockPoolSlice {
}
}
+ this.deleteDuplicateReplicas = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION,
+ DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION_DEFAULT);
+
// Files that were being written when the datanode was last shutdown
// are now moved back to the data directory. It is possible that
// in the future, we might want to do some sort of datanode-local
@@ -515,6 +520,11 @@ class BlockPoolSlice {
final ReplicaInfo replica1, final ReplicaInfo replica2,
final ReplicaMap volumeMap) throws IOException {
+ if (!deleteDuplicateReplicas) {
+ // Leave both block replicas in place.
+ return replica1;
+ }
+
ReplicaInfo replicaToKeep;
ReplicaInfo replicaToDelete;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7062e8d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java
index 01bfb0d..39d3c96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java
@@ -115,9 +115,12 @@ public class TestDFSFinalize {
* the upgrade. Actually it is ok for those contents to change.
* For now disabling block verification so that the contents are
* not changed.
+ * Disable duplicate replica deletion as the test intentionally
+ * mirrors the contents of storage directories.
*/
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
String[] dataNodeDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7062e8d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
index 104b043..bb00144 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
@@ -229,6 +229,7 @@ public class TestDFSUpgrade {
conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
String[] dataNodeDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
log("Normal NameNode upgrade", numDirs);
UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
@@ -370,6 +371,7 @@ public class TestDFSUpgrade {
{
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
@@ -405,6 +407,7 @@ public class TestDFSUpgrade {
int numDirs = 4;
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
[11/34] git commit: HDFS-6931. Move lazily persisted replicas to
finalized directory on DN startup. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6931. Move lazily persisted replicas to finalized directory on DN startup. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/754ba3a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/754ba3a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/754ba3a4
Branch: refs/heads/branch-2.6
Commit: 754ba3a42dff6c5c9401f023c0b5d6bdb31f73c4
Parents: d973656
Author: arp <ar...@apache.org>
Authored: Thu Aug 28 23:13:46 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/BlockPoolSlice.java | 168 ++++++++++++++++---
.../datanode/fsdataset/impl/FsDatasetImpl.java | 9 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 12 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 18 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 53 +++++-
5 files changed, 210 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/754ba3a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 63cac55..637705a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -94,17 +94,6 @@ class BlockPoolSlice {
}
}
- // Delete all checkpointed replicas on startup.
- // TODO: We can move checkpointed replicas to the finalized dir and delete
- // the copy on RAM_DISK. For now we take the simpler approach.
-
- FileUtil.fullyDelete(lazypersistDir);
- if (!this.lazypersistDir.exists()) {
- if (!this.lazypersistDir.mkdirs()) {
- throw new IOException("Failed to mkdirs " + this.lazypersistDir);
- }
- }
-
// Files that were being written when the datanode was last shutdown
// are now moved back to the data directory. It is possible that
// in the future, we might want to do some sort of datanode-local
@@ -277,6 +266,13 @@ class BlockPoolSlice {
return blockFile;
}
+ /**
+ * Save the given replica to persistent storage.
+ *
+ * @param replicaInfo
+ * @return The saved block file.
+ * @throws IOException
+ */
File lazyPersistReplica(ReplicaInfo replicaInfo) throws IOException {
if (!lazypersistDir.exists() && !lazypersistDir.mkdirs()) {
FsDatasetImpl.LOG.warn("Failed to create " + lazypersistDir);
@@ -311,11 +307,21 @@ class BlockPoolSlice {
- void getVolumeMap(ReplicaMap volumeMap) throws IOException {
+ void getVolumeMap(ReplicaMap volumeMap,
+ final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ throws IOException {
+ // Recover lazy persist replicas, they will be added to the volumeMap
+ // when we scan the finalized directory.
+ if (lazypersistDir.exists()) {
+ int numRecovered = moveLazyPersistReplicasToFinalized(lazypersistDir);
+ FsDatasetImpl.LOG.info(
+ "Recovered " + numRecovered + " replicas from " + lazypersistDir);
+ }
+
// add finalized replicas
- addToReplicasMap(volumeMap, finalizedDir, true);
+ addToReplicasMap(volumeMap, finalizedDir, lazyWriteReplicaMap, true);
// add rbw replicas
- addToReplicasMap(volumeMap, rbwDir, false);
+ addToReplicasMap(volumeMap, rbwDir, lazyWriteReplicaMap, false);
}
/**
@@ -344,18 +350,68 @@ class BlockPoolSlice {
/**
+ * Move replicas in the lazy persist directory to their corresponding locations
+ * in the finalized directory.
+ * @return number of replicas recovered.
+ */
+ private int moveLazyPersistReplicasToFinalized(File source)
+ throws IOException {
+ File files[] = FileUtil.listFiles(source);
+ int numRecovered = 0;
+ for (File file : files) {
+ if (file.isDirectory()) {
+ numRecovered += moveLazyPersistReplicasToFinalized(file);
+ }
+
+ if (Block.isMetaFilename(file.getName())) {
+ File metaFile = file;
+ File blockFile = Block.metaToBlockFile(metaFile);
+ long blockId = Block.filename2id(blockFile.getName());
+ File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
+
+ if (blockFile.exists()) {
+ File targetBlockFile = new File(targetDir, blockFile.getName());
+ File targetMetaFile = new File(targetDir, metaFile.getName());
+
+ if (!targetDir.exists() && !targetDir.mkdirs()) {
+ FsDatasetImpl.LOG.warn("Failed to move " + blockFile + " to " + targetDir);
+ continue;
+ }
+
+ metaFile.renameTo(targetMetaFile);
+ blockFile.renameTo(targetBlockFile);
+
+ if (targetBlockFile.exists() && targetMetaFile.exists()) {
+ ++numRecovered;
+ } else {
+ // Failure should be rare.
+ FsDatasetImpl.LOG.warn("Failed to move " + blockFile + " to " + targetDir);
+ }
+ }
+ }
+ }
+
+ FileUtil.fullyDelete(source);
+ return numRecovered;
+ }
+
+ /**
* Add replicas under the given directory to the volume map
* @param volumeMap the replicas map
* @param dir an input directory
+ * @param lazyWriteReplicaMap Map of replicas on transient
+ * storage.
* @param isFinalized true if the directory has finalized replicas;
* false if the directory has rbw replicas
*/
- void addToReplicasMap(ReplicaMap volumeMap, File dir, boolean isFinalized
- ) throws IOException {
+ void addToReplicasMap(ReplicaMap volumeMap, File dir,
+ final LazyWriteReplicaTracker lazyWriteReplicaMap,
+ boolean isFinalized)
+ throws IOException {
File files[] = FileUtil.listFiles(dir);
for (File file : files) {
if (file.isDirectory()) {
- addToReplicasMap(volumeMap, file, isFinalized);
+ addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
}
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
@@ -413,13 +469,83 @@ class BlockPoolSlice {
}
}
- ReplicaInfo oldReplica = volumeMap.add(bpid, newReplica);
- if (oldReplica != null) {
- FsDatasetImpl.LOG.warn("Two block files with the same block id exist " +
- "on disk: " + oldReplica.getBlockFile() + " and " + file );
+ ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId());
+ if (oldReplica == null) {
+ volumeMap.add(bpid, newReplica);
+ } else {
+ // We have multiple replicas of the same block so decide which one
+ // to keep.
+ newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
+ }
+
+ // If we are retaining a replica on transient storage make sure
+ // it is in the lazyWriteReplicaMap so it can be persisted
+ // eventually.
+ if (newReplica.getVolume().isTransientStorage()) {
+ lazyWriteReplicaMap.addReplica(bpid, blockId, newReplica.getVolume());
+ } else {
+ lazyWriteReplicaMap.discardReplica(bpid, blockId, true);
}
}
}
+
+ /**
+ * This method is invoked during DN startup when volumes are scanned to
+ * build up the volumeMap.
+ *
+ * Given two replicas, decide which one to keep. The preference is as
+ * follows:
+ * 1. Prefer the replica with the higher generation stamp.
+ * 2. If generation stamps are equal, prefer the replica with the
+ * larger on-disk length.
+ * 3. If on-disk length is the same, prefer the replica on persistent
+ * storage volume.
+ * 4. All other factors being equal, keep replica1.
+ *
+ * The other replica is removed from the volumeMap and is deleted from
+ * its storage volume.
+ *
+ * @param replica1
+ * @param replica2
+ * @param volumeMap
+ * @return the replica that is retained.
+ * @throws IOException
+ */
+ private ReplicaInfo resolveDuplicateReplicas(
+ final ReplicaInfo replica1, final ReplicaInfo replica2,
+ final ReplicaMap volumeMap) throws IOException {
+
+ ReplicaInfo replicaToKeep;
+ ReplicaInfo replicaToDelete;
+
+ if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) {
+ replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp()
+ ? replica1 : replica2;
+ } else if (replica1.getNumBytes() != replica2.getNumBytes()) {
+ replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ?
+ replica1 : replica2;
+ } else if (replica1.getVolume().isTransientStorage() &&
+ !replica2.getVolume().isTransientStorage()) {
+ replicaToKeep = replica2;
+ } else {
+ replicaToKeep = replica1;
+ }
+
+ replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1;
+
+ // Update volumeMap.
+ volumeMap.add(bpid, replicaToKeep);
+
+ // Delete the files on disk. Failure here is okay.
+ replicaToDelete.getBlockFile().delete();
+ replicaToDelete.getMetaFile().delete();
+
+ FsDatasetImpl.LOG.info(
+ "resolveDuplicateReplicas keeping " + replicaToKeep.getBlockFile() +
+ ", deleting " + replicaToDelete.getBlockFile());
+
+ return replicaToKeep;
+ }
/**
* Find out the number of bytes in the block that match its crc.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/754ba3a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 5c822e8..797dcc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -275,13 +275,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumes = new FsVolumeList(volsFailed, blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode);
- // TODO: Initialize transientReplicaTracker from blocks on disk.
-
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
addVolume(dataLocations, storage.getStorageDir(idx));
}
cacheManager = new FsDatasetCache(this);
+
+ // Start the lazy writer once we have built the replica maps.
lazyWriter = new Daemon(new LazyWriter(
conf.getInt(DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC)));
@@ -301,7 +301,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
FsVolumeImpl fsVolume = FsVolumeImplAllocator.createVolume(
this, sd.getStorageUuid(), dir, this.conf, storageType);
ReplicaMap tempVolumeMap = new ReplicaMap(this);
- fsVolume.getVolumeMap(tempVolumeMap);
+ fsVolume.getVolumeMap(volumeMap, lazyWriteReplicaTracker);
volumeMap.addAll(tempVolumeMap);
volumes.addVolume(fsVolume);
@@ -2194,7 +2194,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumes.addBlockPool(bpid, conf);
volumeMap.initBlockPool(bpid);
}
- volumes.getAllVolumesMap(bpid, volumeMap);
+ volumes.getAllVolumesMap(bpid, volumeMap, lazyWriteReplicaTracker);
}
@Override
@@ -2444,6 +2444,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.warn("Exception saving replica " + replicaState, ioe);
} finally {
if (!succeeded && replicaState != null) {
+ LOG.warn("Failed to save replica " + replicaState + ". re-enqueueing it.");
lazyWriteReplicaTracker.reenqueueReplica(replicaState);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/754ba3a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 93b5895..217e932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -297,14 +297,18 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
}
- void getVolumeMap(ReplicaMap volumeMap) throws IOException {
+ void getVolumeMap(ReplicaMap volumeMap,
+ final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ throws IOException {
for(BlockPoolSlice s : bpSlices.values()) {
- s.getVolumeMap(volumeMap);
+ s.getVolumeMap(volumeMap, lazyWriteReplicaMap);
}
}
- void getVolumeMap(String bpid, ReplicaMap volumeMap) throws IOException {
- getBlockPoolSlice(bpid).getVolumeMap(volumeMap);
+ void getVolumeMap(String bpid, ReplicaMap volumeMap,
+ final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ throws IOException {
+ getBlockPoolSlice(bpid).getVolumeMap(volumeMap, lazyWriteReplicaMap);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/754ba3a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 9fbc349..82fe35f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -119,7 +119,10 @@ class FsVolumeList {
return remaining;
}
- void getAllVolumesMap(final String bpid, final ReplicaMap volumeMap) throws IOException {
+ void getAllVolumesMap(final String bpid,
+ final ReplicaMap volumeMap,
+ final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ throws IOException {
long totalStartTime = Time.monotonicNow();
final List<IOException> exceptions = Collections.synchronizedList(
new ArrayList<IOException>());
@@ -131,7 +134,7 @@ class FsVolumeList {
FsDatasetImpl.LOG.info("Adding replicas to map for block pool " +
bpid + " on volume " + v + "...");
long startTime = Time.monotonicNow();
- v.getVolumeMap(bpid, volumeMap);
+ v.getVolumeMap(bpid, volumeMap, lazyWriteReplicaMap);
long timeTaken = Time.monotonicNow() - startTime;
FsDatasetImpl.LOG.info("Time to add replicas to map for block pool"
+ " " + bpid + " on volume " + v + ": " + timeTaken + "ms");
@@ -160,17 +163,6 @@ class FsVolumeList {
+ totalTimeTaken + "ms");
}
- void getVolumeMap(String bpid, FsVolumeImpl volume, ReplicaMap volumeMap)
- throws IOException {
- FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid +
- " on volume " + volume + "...");
- long startTime = Time.monotonicNow();
- volume.getVolumeMap(bpid, volumeMap);
- long timeTaken = Time.monotonicNow() - startTime;
- FsDatasetImpl.LOG.info("Time to add replicas to map for block pool " + bpid +
- " on volume " + volume + ": " + timeTaken + "ms");
- }
-
/**
* Calls {@link FsVolumeImpl#checkDirs()} on each volume, removing any
* volumes from the active list that result in a DiskErrorException.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/754ba3a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index af0e8ac..cac99a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -265,7 +265,9 @@ public class TestLazyPersistFiles {
LocatedBlocks locatedBlocks = ensureFileReplicasOnStorageType(path, RAM_DISK);
// Sleep for a short time to allow the lazy writer thread to do its job
- Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+ Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ LOG.info("Verifying copy was saved to lazyPersist/");
// Make sure that there is a saved copy of the replica on persistent
// storage.
@@ -330,23 +332,52 @@ public class TestLazyPersistFiles {
ensureFileReplicasOnStorageType(path1, DEFAULT);
}
- /**
- * TODO: Stub test, to be completed.
- * Verify that checksum computation is skipped for files written to memory.
- */
@Test (timeout=300000)
- public void testChecksumIsSkipped()
+ public void testDnRestartWithSavedReplicas()
throws IOException, InterruptedException {
+
startUpCluster(REPL_FACTOR,
- new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ new StorageType[] {RAM_DISK, DEFAULT },
+ (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
makeTestFile(path1, BLOCK_SIZE, true);
ensureFileReplicasOnStorageType(path1, RAM_DISK);
- // Verify checksum was not computed.
+ // Sleep for a short time to allow the lazy writer thread to do its job.
+ // However the block replica should not be evicted from RAM_DISK yet.
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+ LOG.info("Restarting the DataNode");
+ cluster.restartDataNode(0, true);
+ cluster.waitActive();
+
+ // Ensure that the replica is now on persistent storage.
+ ensureFileReplicasOnStorageType(path1, DEFAULT);
+ }
+
+ @Test (timeout=300000)
+ public void testDnRestartWithUnsavedReplicas()
+ throws IOException, InterruptedException {
+
+ startUpCluster(REPL_FACTOR,
+ new StorageType[] {RAM_DISK, DEFAULT },
+ (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ stopLazyWriter(cluster.getDataNodes().get(0));
+
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ makeTestFile(path1, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ LOG.info("Restarting the DataNode");
+ cluster.restartDataNode(0, true);
+ cluster.waitActive();
+
+ // Ensure that the replica is still on transient storage.
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
}
// ---- Utility functions for all test cases -------------------------------
@@ -443,4 +474,10 @@ public class TestLazyPersistFiles {
return locatedBlocks;
}
+
+ private void stopLazyWriter(DataNode dn) {
+ // Stop the lazyWriter daemon.
+ FsDatasetImpl fsDataset = ((FsDatasetImpl) dn.getFSDataset());
+ ((FsDatasetImpl.LazyWriter) fsDataset.lazyWriter.getRunnable()).stop();
+ }
}
[06/34] git commit: HDFS-6925. DataNode should attempt to place
replicas on transient storage first if lazyPersist flag is received. (Arpit
Agarwal)
Posted by ji...@apache.org.
HDFS-6925. DataNode should attempt to place replicas on transient storage first if lazyPersist flag is received. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34d0088b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34d0088b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34d0088b
Branch: refs/heads/branch-2.6
Commit: 34d0088bf9d793de888c8dd969a047dbae77cfb5
Parents: 8c3c0ec
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:35:47 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockReceiver.java | 5 +-
.../hdfs/server/datanode/DataXceiver.java | 6 +-
.../hdfs/server/datanode/DirectoryScanner.java | 2 +-
.../hadoop/hdfs/server/datanode/Replica.java | 5 ++
.../hdfs/server/datanode/ReplicaInfo.java | 30 ++--------
.../server/datanode/ReplicaUnderRecovery.java | 3 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 2 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 4 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 5 +-
.../RoundRobinVolumeChoosingPolicy.java | 13 ++++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 49 ++++++++++++----
.../fsdataset/impl/FsTransientVolumeImpl.java | 60 ++++++++++++++++++++
.../datanode/fsdataset/impl/FsVolumeImpl.java | 12 +++-
.../fsdataset/impl/FsVolumeImplAllocator.java | 44 ++++++++++++++
.../datanode/fsdataset/impl/FsVolumeList.java | 20 ++++++-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 5 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 10 +++-
.../hdfs/server/datanode/TestBlockRecovery.java | 4 +-
.../server/datanode/TestDirectoryScanner.java | 6 +-
.../server/datanode/TestSimulatedFSDataset.java | 2 +-
.../fsdataset/impl/FsDatasetTestUtil.java | 2 +-
.../fsdataset/impl/TestWriteToReplica.java | 12 ++--
23 files changed, 231 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index bfb2233..4d1cc6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -139,7 +139,8 @@ class BlockReceiver implements Closeable {
final long newGs, final long minBytesRcvd, final long maxBytesRcvd,
final String clientname, final DatanodeInfo srcDataNode,
final DataNode datanode, DataChecksum requestedChecksum,
- CachingStrategy cachingStrategy) throws IOException {
+ CachingStrategy cachingStrategy,
+ final boolean allowLazyPersist) throws IOException {
try{
this.block = block;
this.in = in;
@@ -180,7 +181,7 @@ class BlockReceiver implements Closeable {
} else {
switch (stage) {
case PIPELINE_SETUP_CREATE:
- replicaInfo = datanode.data.createRbw(storageType, block);
+ replicaInfo = datanode.data.createRbw(storageType, block, allowLazyPersist);
datanode.notifyNamenodeReceivingBlock(
block, replicaInfo.getStorageUuid());
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 3b8304e..67eb941 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -607,8 +607,8 @@ class DataXceiver extends Receiver implements Runnable {
peer.getLocalAddressString(),
stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd,
clientname, srcDataNode, datanode, requestedChecksum,
- cachingStrategy);
-
+ cachingStrategy, allowLazyPersist);
+
storageUuid = blockReceiver.getStorageUuid();
} else {
storageUuid = datanode.data.recoverClose(
@@ -1048,7 +1048,7 @@ class DataXceiver extends Receiver implements Runnable {
proxyReply, proxySock.getRemoteSocketAddress().toString(),
proxySock.getLocalSocketAddress().toString(),
null, 0, 0, 0, "", null, datanode, remoteChecksum,
- CachingStrategy.newDropBehind());
+ CachingStrategy.newDropBehind(), false);
// receive a block
blockReceiver.receiveBlock(null, null, replyOut, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index a47f2ef..c313b04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -399,7 +399,7 @@ public class DirectoryScanner implements Runnable {
/**
* Reconcile differences between disk and in-memory blocks
*/
- void reconcile() {
+ void reconcile() throws IOException {
scan();
for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
String bpid = entry.getKey();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
index a480bb1..b6e5ba9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
@@ -59,4 +59,9 @@ public interface Replica {
* Return the storageUuid of the volume that stores this replica.
*/
public String getStorageUuid();
+
+ /**
+ * Return true if the target volume is backed by RAM.
+ */
+ public boolean isOnTransientStorage();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
index 49ac605..940d3eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
@@ -62,17 +62,6 @@ abstract public class ReplicaInfo extends Block implements Replica {
private static final Map<String, File> internedBaseDirs = new HashMap<String, File>();
/**
- * Constructor for a zero length replica
- * @param blockId block id
- * @param genStamp replica generation stamp
- * @param vol volume where replica is located
- * @param dir directory path where block and meta files are located
- */
- ReplicaInfo(long blockId, long genStamp, FsVolumeSpi vol, File dir) {
- this( blockId, 0L, genStamp, vol, dir);
- }
-
- /**
* Constructor
* @param block a block
* @param vol volume where replica is located
@@ -296,20 +285,6 @@ abstract public class ReplicaInfo extends Block implements Replica {
return true;
}
- /**
- * Set this replica's generation stamp to be a newer one
- * @param newGS new generation stamp
- * @throws IOException is the new generation stamp is not greater than the current one
- */
- void setNewerGenerationStamp(long newGS) throws IOException {
- long curGS = getGenerationStamp();
- if (newGS <= curGS) {
- throw new IOException("New generation stamp (" + newGS
- + ") must be greater than current one (" + curGS + ")");
- }
- setGenerationStamp(newGS);
- }
-
@Override //Object
public String toString() {
return getClass().getSimpleName()
@@ -321,4 +296,9 @@ abstract public class ReplicaInfo extends Block implements Replica {
+ "\n getVolume() = " + getVolume()
+ "\n getBlockFile() = " + getBlockFile();
}
+
+ @Override
+ public boolean isOnTransientStorage() {
+ return volume.isTransientStorage();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java
index 35f7c93..2cd8a01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java
@@ -37,8 +37,7 @@ public class ReplicaUnderRecovery extends ReplicaInfo {
// that the replica will be bumped to after recovery
public ReplicaUnderRecovery(ReplicaInfo replica, long recoveryId) {
- super(replica.getBlockId(), replica.getNumBytes(), replica.getGenerationStamp(),
- replica.getVolume(), replica.getDir());
+ super(replica, replica.getVolume(), replica.getDir());
if ( replica.getState() != ReplicaState.FINALIZED &&
replica.getState() != ReplicaState.RBW &&
replica.getState() != ReplicaState.RWR ) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index ec19ec5..235cd7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -99,7 +99,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
@Override
public synchronized V chooseVolume(List<V> volumes,
- final long replicaSize) throws IOException {
+ long replicaSize) throws IOException {
if (volumes.size() < 1) {
throw new DiskOutOfSpaceException("No more available volumes");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 4c03151..4a5580c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -122,7 +122,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* as corrupted.
*/
public void checkAndUpdate(String bpid, long blockId, File diskFile,
- File diskMetaFile, FsVolumeSpi vol);
+ File diskMetaFile, FsVolumeSpi vol) throws IOException;
/**
* @param b - the block
@@ -197,7 +197,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* @throws IOException if an error occurs
*/
public ReplicaInPipelineInterface createRbw(StorageType storageType,
- ExtendedBlock b) throws IOException;
+ ExtendedBlock b, boolean allowLazyPersist) throws IOException;
/**
* Recovers a RBW replica and returns the meta info of the replica
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index cba23c3..4f45922 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -56,4 +56,7 @@ public interface FsVolumeSpi {
* Release disk space previously reserved for RBW block.
*/
public void releaseReservedSpace(long bytesToRelease);
-}
\ No newline at end of file
+
+ /** Returns true if the volume is NOT backed by persistent storage. */
+ public boolean isTransientStorage();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
index 7f4bdae..55a3560 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset;
import java.io.IOException;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
/**
@@ -27,12 +30,14 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
*/
public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
implements VolumeChoosingPolicy<V> {
+ public static final Log LOG = LogFactory.getLog(RoundRobinVolumeChoosingPolicy.class);
private int curVolume = 0;
@Override
- public synchronized V chooseVolume(final List<V> volumes, final long blockSize
- ) throws IOException {
+ public synchronized V chooseVolume(final List<V> volumes, long blockSize)
+ throws IOException {
+
if(volumes.size() < 1) {
throw new DiskOutOfSpaceException("No more available volumes");
}
@@ -50,7 +55,9 @@ public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
final V volume = volumes.get(curVolume);
curVolume = (curVolume + 1) % volumes.size();
long availableVolumeSize = volume.getAvailable();
- if (availableVolumeSize > blockSize) { return volume; }
+ if (availableVolumeSize > blockSize) {
+ return volume;
+ }
if (availableVolumeSize > maxAvailable) {
maxAvailable = availableVolumeSize;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 57706f8..701207d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -45,6 +45,9 @@ import javax.management.ObjectName;
import javax.management.StandardMBean;
import com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.TreeMultimap;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -278,7 +281,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// If IOException raises from FsVolumeImpl() or getVolumeMap(), there is
// nothing needed to be rolled back to make various data structures, e.g.,
// storageMap and asyncDiskService, consistent.
- FsVolumeImpl fsVolume = new FsVolumeImpl(
+ FsVolumeImpl fsVolume = FsVolumeImplAllocator.createVolume(
this, sd.getStorageUuid(), dir, this.conf, storageType);
ReplicaMap tempVolumeMap = new ReplicaMap(this);
fsVolume.getVolumeMap(tempVolumeMap);
@@ -550,16 +553,16 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
* Get File name for a given block.
*/
private File getBlockFile(ExtendedBlock b) throws IOException {
- return getBlockFile(b.getBlockPoolId(), b.getLocalBlock());
+ return getBlockFile(b.getBlockPoolId(), b.getBlockId());
}
/**
* Get File name for a given block.
*/
- File getBlockFile(String bpid, Block b) throws IOException {
- File f = validateBlockFile(bpid, b);
+ File getBlockFile(String bpid, long blockId) throws IOException {
+ File f = validateBlockFile(bpid, blockId);
if(f == null) {
- throw new IOException("Block " + b + " is not valid.");
+ throw new IOException("BlockId " + blockId + " is not valid.");
}
return f;
}
@@ -949,8 +952,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public synchronized ReplicaInPipeline createRbw(StorageType storageType,
- ExtendedBlock b) throws IOException {
- ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
+ ExtendedBlock b, boolean allowLazyPersist) throws IOException {
+ ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
b.getBlockId());
if (replicaInfo != null) {
throw new ReplicaAlreadyExistsException("Block " + b +
@@ -958,8 +961,25 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
" and thus cannot be created.");
}
// create a new block
- FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
- // create a rbw file to hold block in the designated volume
+ FsVolumeImpl v;
+ while (true) {
+ try {
+ if (allowLazyPersist) {
+ // First try to place the block on a transient volume.
+ v = volumes.getNextTransientVolume(b.getNumBytes());
+ } else {
+ v = volumes.getNextVolume(storageType, b.getNumBytes());
+ }
+ } catch (DiskOutOfSpaceException de) {
+ if (allowLazyPersist) {
+ allowLazyPersist = false;
+ continue;
+ }
+ throw de;
+ }
+ break;
+ }
+ // create an rbw file to hold block in the designated volume
File f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(),
b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
@@ -1321,11 +1341,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
/**
* Find the file corresponding to the block and return it if it exists.
*/
- File validateBlockFile(String bpid, Block b) {
+ File validateBlockFile(String bpid, long blockId) {
//Should we check for metadata file too?
final File f;
synchronized(this) {
- f = getFile(bpid, b.getBlockId());
+ f = getFile(bpid, blockId);
}
if(f != null ) {
@@ -1337,7 +1357,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("b=" + b + ", f=" + f);
+ LOG.debug("blockId=" + blockId + ", f=" + f);
}
return null;
}
@@ -1497,6 +1517,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
": volume was not an instance of FsVolumeImpl.");
return;
}
+ if (volume.isTransientStorage()) {
+ LOG.warn("Caching not supported on block with id " + blockId +
+ " since the volume is backed by RAM.");
+ return;
+ }
success = true;
} finally {
if (!success) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
new file mode 100644
index 0000000..dafa74f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.StorageType;
+
+/**
+ * Volume for storing replicas in memory. These can be deleted at any time
+ * to make space for new replicas and there is no persistence guarantee.
+ *
+ * The backing store for these replicas is expected to be RAM_DISK.
+ * The backing store may be disk when testing.
+ *
+ * It uses the {@link FsDatasetImpl} object for synchronization.
+ */
+@InterfaceAudience.Private
+@VisibleForTesting
+public class FsTransientVolumeImpl extends FsVolumeImpl {
+
+
+ FsTransientVolumeImpl(FsDatasetImpl dataset, String storageID, File currentDir,
+ Configuration conf, StorageType storageType)
+ throws IOException {
+ super(dataset, storageID, currentDir, conf, storageType);
+ }
+
+ @Override
+ protected ThreadPoolExecutor initializeCacheExecutor(File parent) {
+ // Can't 'cache' replicas already in RAM.
+ return null;
+ }
+
+ @Override
+ public boolean isTransientStorage() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 40c6840..4c0b5f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -77,7 +77,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
* dfs.datanode.fsdatasetcache.max.threads.per.volume) to limit resource
* contention.
*/
- private final ThreadPoolExecutor cacheExecutor;
+ protected ThreadPoolExecutor cacheExecutor;
FsVolumeImpl(FsDatasetImpl dataset, String storageID, File currentDir,
Configuration conf, StorageType storageType) throws IOException {
@@ -202,6 +202,11 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
@Override
+ public boolean isTransientStorage() {
+ return false;
+ }
+
+ @Override
public String getPath(String bpid) throws IOException {
return getBlockPoolSlice(bpid).getDirectory().getAbsolutePath();
}
@@ -324,7 +329,9 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
void shutdown() {
- cacheExecutor.shutdown();
+ if (cacheExecutor != null) {
+ cacheExecutor.shutdown();
+ }
Set<Entry<String, BlockPoolSlice>> set = bpSlices.entrySet();
for (Entry<String, BlockPoolSlice> entry : set) {
entry.getValue().shutdown();
@@ -417,6 +424,5 @@ public class FsVolumeImpl implements FsVolumeSpi {
DatanodeStorage toDatanodeStorage() {
return new DatanodeStorage(storageID, DatanodeStorage.State.NORMAL, storageType);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
new file mode 100644
index 0000000..14d3aaf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.StorageType;
+
+/**
+ * Generate volumes based on the storageType.
+ */
+@InterfaceAudience.Private
+class FsVolumeImplAllocator {
+ static FsVolumeImpl createVolume(FsDatasetImpl fsDataset, String storageUuid,
+ File dir, Configuration conf, StorageType storageType)
+ throws IOException {
+ switch(storageType) {
+ case RAM_DISK:
+ return new FsTransientVolumeImpl(
+ fsDataset, storageUuid, dir, conf, storageType);
+ default:
+ return new FsVolumeImpl(
+ fsDataset, storageUuid, dir, conf, storageType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 90739c3..9fbc349 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -68,7 +68,25 @@ class FsVolumeList {
}
return blockChooser.chooseVolume(list, blockSize);
}
-
+
+ /**
+ * Get next volume. Synchronized to ensure {@link #curVolume} is updated
+ * by a single thread and next volume is chosen with no concurrent
+ * update to {@link #volumes}.
+ * @param blockSize free space needed on the volume
+ * @return next volume to store the block in.
+ */
+ synchronized FsVolumeImpl getNextTransientVolume(
+ long blockSize) throws IOException {
+ final List<FsVolumeImpl> list = new ArrayList<FsVolumeImpl>(volumes.size());
+ for(FsVolumeImpl v : volumes) {
+ if (v.isTransientStorage()) {
+ list.add(v);
+ }
+ }
+ return blockChooser.chooseVolume(list, blockSize);
+ }
+
long getDfsUsed() throws IOException {
long dfsUsed = 0L;
for (FsVolumeImpl v : volumes) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
index 29ac3f2..b7fdccf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
@@ -98,9 +98,10 @@ public class TestWriteBlockGetsBlockLengthHint {
*/
@Override
public synchronized ReplicaInPipelineInterface createRbw(
- StorageType storageType, ExtendedBlock b) throws IOException {
+ StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
+ throws IOException {
assertThat(b.getLocalBlock().getNumBytes(), is(EXPECTED_BLOCK_LENGTH));
- return super.createRbw(storageType, b);
+ return super.createRbw(storageType, b, allowLazyPersist);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
index 9ea6c51..e9557da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
@@ -364,7 +364,7 @@ public abstract class BlockReportTestBase {
// Create a bogus new block which will not be present on the namenode.
ExtendedBlock b = new ExtendedBlock(
poolId, rand.nextLong(), 1024L, rand.nextLong());
- dn.getFSDataset().createRbw(StorageType.DEFAULT, b);
+ dn.getFSDataset().createRbw(StorageType.DEFAULT, b, false);
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 83d93f0..eab599d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -300,6 +300,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
public ChunkChecksum getLastChecksumAndDataLen() {
return new ChunkChecksum(oStream.getLength(), null);
}
+
+ @Override
+ public boolean isOnTransientStorage() {
+ return false;
+ }
}
/**
@@ -747,7 +752,8 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override // FsDatasetSpi
public synchronized ReplicaInPipelineInterface createRbw(
- StorageType storageType, ExtendedBlock b) throws IOException {
+ StorageType storageType, ExtendedBlock b,
+ boolean allowLazyPersist) throws IOException {
return createTemporary(storageType, b);
}
@@ -1083,7 +1089,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override
public void checkAndUpdate(String bpid, long blockId, File diskFile,
- File diskMetaFile, FsVolumeSpi vol) {
+ File diskMetaFile, FsVolumeSpi vol) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index f627c42..4dfc773 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -529,7 +529,7 @@ public class TestBlockRecovery {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
}
- dn.data.createRbw(StorageType.DEFAULT, block);
+ dn.data.createRbw(StorageType.DEFAULT, block, false);
try {
dn.syncBlock(rBlock, initBlockRecords(dn));
fail("Sync should fail");
@@ -553,7 +553,7 @@ public class TestBlockRecovery {
LOG.debug("Running " + GenericTestUtils.getMethodName());
}
ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(
- StorageType.DEFAULT, block);
+ StorageType.DEFAULT, block, false);
ReplicaOutputStreams streams = null;
try {
streams = replicaInfo.createStreams(true,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index bc50eaa..78b9e2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -215,7 +215,7 @@ public class TestDirectoryScanner {
}
private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile,
- long missingMemoryBlocks, long mismatchBlocks) {
+ long missingMemoryBlocks, long mismatchBlocks) throws IOException {
scanner.reconcile();
assertTrue(scanner.diffs.containsKey(bpid));
@@ -431,6 +431,10 @@ public class TestDirectoryScanner {
@Override
public void releaseReservedSpace(long bytesToRelease) {
+
+ @Override
+ public boolean isTransientStorage() {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index bd6c3de..099a0cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -67,7 +67,7 @@ public class TestSimulatedFSDataset {
// we pass expected len as zero, - fsdataset should use the sizeof actual
// data written
ReplicaInPipelineInterface bInfo = fsdataset.createRbw(
- StorageType.DEFAULT, b);
+ StorageType.DEFAULT, b, false);
ReplicaOutputStreams out = bInfo.createStreams(true,
DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512));
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
index 6bd36ed..d6d7dd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
@@ -34,7 +34,7 @@ public class FsDatasetTestUtil {
public static File getBlockFile(FsDatasetSpi<?> fsd, String bpid, Block b
) throws IOException {
- return ((FsDatasetImpl)fsd).getBlockFile(bpid, b);
+ return ((FsDatasetImpl)fsd).getBlockFile(bpid, b.getBlockId());
}
public static File getMetaFile(FsDatasetSpi<?> fsd, String bpid, Block b)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34d0088b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index a870aa9..60c6d03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -358,7 +358,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[FINALIZED]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[FINALIZED], false);
Assert.fail("Should not have created a replica that's already " +
"finalized " + blocks[FINALIZED]);
} catch (ReplicaAlreadyExistsException e) {
@@ -376,7 +376,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[TEMPORARY]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[TEMPORARY], false);
Assert.fail("Should not have created a replica that had created as " +
"temporary " + blocks[TEMPORARY]);
} catch (ReplicaAlreadyExistsException e) {
@@ -386,7 +386,7 @@ public class TestWriteToReplica {
0L, blocks[RBW].getNumBytes()); // expect to be successful
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RBW]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[RBW], false);
Assert.fail("Should not have created a replica that had created as RBW " +
blocks[RBW]);
} catch (ReplicaAlreadyExistsException e) {
@@ -402,7 +402,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RWR]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[RWR], false);
Assert.fail("Should not have created a replica that was waiting to be " +
"recovered " + blocks[RWR]);
} catch (ReplicaAlreadyExistsException e) {
@@ -418,7 +418,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RUR]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[RUR], false);
Assert.fail("Should not have created a replica that was under recovery " +
blocks[RUR]);
} catch (ReplicaAlreadyExistsException e) {
@@ -435,7 +435,7 @@ public class TestWriteToReplica {
e.getMessage().contains(ReplicaNotFoundException.NON_EXISTENT_REPLICA));
}
- dataSet.createRbw(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ dataSet.createRbw(StorageType.DEFAULT, blocks[NON_EXISTENT], false);
}
private void testWriteToTemporary(FsDatasetImpl dataSet, ExtendedBlock[] blocks) throws IOException {
[02/34] git commit: HDFS-6922. Add LazyPersist flag to INodeFile,
save it in FsImage and edit logs. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6922. Add LazyPersist flag to INodeFile, save it in FsImage and edit logs. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d9d29ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d9d29ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d9d29ad
Branch: refs/heads/branch-2.6
Commit: 1d9d29ad40dd40bc0db056aca7247f1ce7f4e33b
Parents: d09f316
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:12:19 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:48 2014 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockCollection.java | 6 +++++
.../hdfs/server/namenode/FSDirectory.java | 16 +++++++++-----
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 ++
.../hdfs/server/namenode/FSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/FSEditLogOp.java | 23 ++++++++++++++++++++
.../hdfs/server/namenode/FSImageFormat.java | 9 ++++++--
.../server/namenode/FSImageFormatPBINode.java | 5 +++--
.../server/namenode/FSImageSerialization.java | 6 +++--
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 14 +++++++++---
.../server/namenode/INodeFileAttributes.java | 10 ++++++---
.../server/namenode/NameNodeLayoutVersion.java | 2 ++
.../snapshot/FSImageFormatPBSnapshot.java | 5 ++++-
.../tools/offlineImageViewer/FSImageLoader.java | 1 +
.../offlineImageViewer/PBImageXmlWriter.java | 4 ++++
.../hadoop-hdfs/src/main/proto/fsimage.proto | 1 +
.../hdfs/server/namenode/CreateEditsLog.java | 4 ++--
.../hdfs/server/namenode/TestEditLog.java | 2 +-
.../namenode/TestFSPermissionChecker.java | 2 +-
.../hdfs/server/namenode/TestINodeFile.java | 8 +++----
20 files changed, 96 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index d095a1b..c97dcb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -55,6 +55,12 @@ public interface BlockCollection {
public long getPreferredBlockSize();
/**
+ * Return true if the file was created with {@Link CreateFlag#LAZY_PERSIST}.
+ * @return
+ */
+ public boolean getLazyPersistFlag();
+
+ /**
* Get block replication for the collection
* @return block replication value
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 083816e..c2dfeae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -299,13 +299,14 @@ public class FSDirectory implements Closeable {
*/
INodeFile addFile(String path, PermissionStatus permissions,
short replication, long preferredBlockSize,
+ boolean isLazyPersist,
String clientName, String clientMachine)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException, AclException {
long modTime = now();
INodeFile newNode = newINodeFile(namesystem.allocateNewInodeId(),
- permissions, modTime, modTime, replication, preferredBlockSize);
+ permissions, modTime, modTime, replication, preferredBlockSize, isLazyPersist);
newNode.toUnderConstruction(clientName, clientMachine);
boolean added = false;
@@ -335,6 +336,7 @@ public class FSDirectory implements Closeable {
long modificationTime,
long atime,
long preferredBlockSize,
+ boolean isLazyPersist,
boolean underConstruction,
String clientName,
String clientMachine,
@@ -343,12 +345,12 @@ public class FSDirectory implements Closeable {
assert hasWriteLock();
if (underConstruction) {
newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize, storagePolicyId);
+ modificationTime, replication, preferredBlockSize, storagePolicyId, isLazyPersist);
newNode.toUnderConstruction(clientName, clientMachine);
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize, storagePolicyId);
+ replication, preferredBlockSize, storagePolicyId, isLazyPersist);
}
try {
@@ -2408,6 +2410,7 @@ public class FSDirectory implements Closeable {
final FileEncryptionInfo feInfo = isRawPath ? null :
getFileEncryptionInfo(node, snapshot, iip);
+ boolean isLazyPersist = false;
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -2415,6 +2418,7 @@ public class FSDirectory implements Closeable {
blocksize = fileNode.getPreferredBlockSize();
isEncrypted = (feInfo != null) ||
(isRawPath && isInAnEZ(INodesInPath.fromINode(node)));
+ isLazyPersist = fileNode.getLazyPersistFlag();
} else {
isEncrypted = isInAnEZ(INodesInPath.fromINode(node));
}
@@ -2427,6 +2431,7 @@ public class FSDirectory implements Closeable {
node.isDirectory(),
replication,
blocksize,
+ isLazyPersist,
node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot, isEncrypted),
@@ -2450,6 +2455,7 @@ public class FSDirectory implements Closeable {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
+ boolean isLazyPersist = false;
LocatedBlocks loc = null;
final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
@@ -2458,7 +2464,7 @@ public class FSDirectory implements Closeable {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
replication = fileNode.getFileReplication(snapshot);
- blocksize = fileNode.getPreferredBlockSize();
+ isLazyPersist = fileNode.getLazyPersistFlag();
final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID;
final boolean isUc = !inSnapshot && fileNode.isUnderConstruction();
@@ -2481,7 +2487,7 @@ public class FSDirectory implements Closeable {
HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
- blocksize, node.getModificationTime(snapshot),
+ blocksize, isLazyPersist, node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot, isEncrypted),
node.getUserName(snapshot), node.getGroupName(snapshot),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index cb44011..1a57179 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -714,6 +714,7 @@ public class FSEditLog implements LogsPurgeable {
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
+ .setLazyPersistFlag(newNode.getLazyPersistFlag())
.setBlocks(newNode.getBlocks())
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
@@ -746,6 +747,7 @@ public class FSEditLog implements LogsPurgeable {
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
+ .setLazyPersistFlag(newNode.getLazyPersistFlag())
.setBlocks(newNode.getBlocks())
.setPermissionStatus(newNode.getPermissionStatus());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 7dfe688..472a7f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -366,7 +366,7 @@ public class FSEditLogLoader {
path, addCloseOp.permissions, addCloseOp.aclEntries,
addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
- addCloseOp.blockSize, true, addCloseOp.clientName,
+ addCloseOp.blockSize, addCloseOp.isLazyPersist, true, addCloseOp.clientName,
addCloseOp.clientMachine, addCloseOp.storagePolicyId);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 63983d7..f66530d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -406,6 +406,7 @@ public abstract class FSEditLogOp {
long mtime;
long atime;
long blockSize;
+ boolean isLazyPersist;
Block[] blocks;
PermissionStatus permissions;
List<AclEntry> aclEntries;
@@ -455,6 +456,11 @@ public abstract class FSEditLogOp {
return (T)this;
}
+ <T extends AddCloseOp> T setLazyPersistFlag(boolean isLazyPersist) {
+ this.isLazyPersist = isLazyPersist;
+ return (T)this;
+ }
+
<T extends AddCloseOp> T setBlocks(Block[] blocks) {
if (blocks.length > MAX_BLOCKS) {
throw new RuntimeException("Can't have more than " + MAX_BLOCKS +
@@ -512,6 +518,7 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeLong(mtime, out);
FSImageSerialization.writeLong(atime, out);
FSImageSerialization.writeLong(blockSize, out);
+ FSImageSerialization.writeInt((isLazyPersist ? 1 : 0), out);
new ArrayWritable(Block.class, blocks).write(out);
permissions.write(out);
@@ -581,6 +588,13 @@ public abstract class FSEditLogOp {
this.blockSize = readLong(in);
}
+ if (NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.LAZY_PERSIST_FILES, logVersion)) {
+ this.isLazyPersist = (FSImageSerialization.readInt(in) != 0);
+ } else {
+ this.isLazyPersist = false;
+ }
+
this.blocks = readBlocks(in, logVersion);
this.permissions = PermissionStatus.read(in);
@@ -646,6 +660,8 @@ public abstract class FSEditLogOp {
builder.append(atime);
builder.append(", blockSize=");
builder.append(blockSize);
+ builder.append(", lazyPersist");
+ builder.append(isLazyPersist);
builder.append(", blocks=");
builder.append(Arrays.toString(blocks));
builder.append(", permissions=");
@@ -686,6 +702,8 @@ public abstract class FSEditLogOp {
Long.toString(atime));
XMLUtils.addSaxString(contentHandler, "BLOCKSIZE",
Long.toString(blockSize));
+ XMLUtils.addSaxString(contentHandler, "LAZY_PERSIST",
+ Boolean.toString(isLazyPersist));
XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName);
XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine);
XMLUtils.addSaxString(contentHandler, "OVERWRITE",
@@ -711,6 +729,11 @@ public abstract class FSEditLogOp {
this.mtime = Long.parseLong(st.getValue("MTIME"));
this.atime = Long.parseLong(st.getValue("ATIME"));
this.blockSize = Long.parseLong(st.getValue("BLOCKSIZE"));
+
+ String lazyPersistString = st.getValueOrNull("LAZY_PERSIST");
+ this.isLazyPersist =
+ lazyPersistString != null && Boolean.parseBoolean(lazyPersistString);
+
this.clientName = st.getValue("CLIENT_NAME");
this.clientMachine = st.getValue("CLIENT_MACHINE");
this.overwrite = Boolean.parseBoolean(st.getValueOrNull("OVERWRITE"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 5c0e292..6d050e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -785,8 +785,11 @@ public class FSImageFormat {
if (counter != null) {
counter.increment();
}
+
+ // Images in the old format will not have the lazyPersist flag so it is
+ // safe to pass false always.
final INodeFile file = new INodeFile(inodeId, localName, permissions,
- modificationTime, atime, blocks, replication, blockSize, (byte)0);
+ modificationTime, atime, blocks, replication, blockSize, (byte)0, false);
if (underConstruction) {
file.toUnderConstruction(clientName, clientMachine);
}
@@ -889,8 +892,10 @@ public class FSImageFormat {
in.readShort());
final long preferredBlockSize = in.readLong();
+ // LazyPersist flag will not be present in old image formats and hence
+ // can be safely set to false always.
return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
- accessTime, replication, preferredBlockSize, (byte)0, null);
+ accessTime, replication, preferredBlockSize, (byte)0, false null);
}
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 5631b2b..8cce6d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -290,7 +290,7 @@ public final class FSImageFormatPBINode {
final INodeFile file = new INodeFile(n.getId(),
n.getName().toByteArray(), permissions, f.getModificationTime(),
f.getAccessTime(), blocks, replication, f.getPreferredBlockSize(),
- (byte)f.getStoragePolicyID());
+ (byte)f.getStoragePolicyID(), f.hasIsLazyPersist() ? f.getIsLazyPersist() : false);
if (f.hasAcl()) {
file.addAclFeature(new AclFeature(loadAclEntries(f.getAcl(),
@@ -400,7 +400,8 @@ public final class FSImageFormatPBINode {
.setPermission(buildPermissionStatus(file, state.getStringMap()))
.setPreferredBlockSize(file.getPreferredBlockSize())
.setReplication(file.getFileReplication())
- .setStoragePolicyID(file.getLocalStoragePolicyID());
+ .setStoragePolicyID(file.getLocalStoragePolicyID())
+ .setIsLazyPersist(file.getLazyPersistFlag());
AclFeature f = file.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index d3472fc..1872898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -148,14 +148,16 @@ public class FSImageSerialization {
int numLocs = in.readInt();
assert numLocs == 0 : "Unexpected block locations";
+ // Images in the pre-protobuf format will not have the lazyPersist flag,
+ // so it is safe to pass false always.
INodeFile file = new INodeFile(inodeId, name, perm, modificationTime,
- modificationTime, blocks, blockReplication, preferredBlockSize, (byte)0);
+ modificationTime, blocks, blockReplication, preferredBlockSize, (byte)0, false);
file.toUnderConstruction(clientName, clientMachine);
return file;
}
// Helper function that writes an INodeUnderConstruction
- // into the input stream
+ // into the output stream
//
static void writeINodeUnderConstruction(DataOutputStream out, INodeFile cons,
String path) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index f3446ea..fdf4f2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2710,7 +2710,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if (parent != null && mkdirsRecursively(parent.toString(),
permissions, true, now())) {
newNode = dir.addFile(src, permissions, replication, blockSize,
- holder, clientMachine);
+ isLazyPersist, holder, clientMachine);
}
if (newNode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 583c193..05e7bd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -80,7 +80,8 @@ public class INodeFile extends INodeWithAdditionalFields
PREFERRED_BLOCK_SIZE(null, 48, 1),
REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 12, 1),
STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH,
- 0);
+ 0),
+ LAZY_PERSIST(REPLICATION.BITS, 4, 0);
private final LongBitFormat BITS;
@@ -106,8 +107,12 @@ public class INodeFile extends INodeWithAdditionalFields
h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
h = REPLICATION.BITS.combine(replication, h);
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
+ h = LAZY_PERSIST.BITS.combine(isLazyPersist ? 1 : 0, h);
return h;
}
+ static boolean getLazyPersistFlag(long header) {
+ return LAZY_PERSIST.BITS.retrieve(header) == 0 ? false : true;
+ }
}
private long header = 0L;
@@ -116,10 +121,10 @@ public class INodeFile extends INodeWithAdditionalFields
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
long atime, BlockInfo[] blklist, short replication,
- long preferredBlockSize, byte storagePolicyID) {
+ long preferredBlockSize, byte storagePolicyID, boolean isLazyPersist) {
super(id, name, permissions, mtime, atime);
header = HeaderFormat.toLong(preferredBlockSize, replication,
- storagePolicyID);
+ storagePolicyID, isLazyPersist);
this.blocks = blklist;
}
@@ -392,6 +397,9 @@ public class INodeFile extends INodeWithAdditionalFields
recordModification(latestSnapshotId);
setStoragePolicyID(storagePolicyId);
}
+ public boolean getLazyPersistFlag() {
+ return HeaderFormat.getLazyPersistFlag(header);
+ }
@Override
public long getHeaderLong() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
index f9d2700..05ce78f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.server.namenode.INodeFile.HeaderFormat;
import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
-
/**
* The attributes of a file.
*/
@@ -33,6 +32,8 @@ public interface INodeFileAttributes extends INodeAttributes {
/** @return preferred block size in bytes */
public long getPreferredBlockSize();
+ public boolean getLazyPersistFlag();
+
/** @return the header as a long. */
public long getHeaderLong();
@@ -48,10 +49,10 @@ public interface INodeFileAttributes extends INodeAttributes {
public SnapshotCopy(byte[] name, PermissionStatus permissions,
AclFeature aclFeature, long modificationTime, long accessTime,
short replication, long preferredBlockSize, byte storagePolicyID,
- XAttrFeature xAttrsFeature) {
+ boolean isTransient, XAttrFeature xAttrsFeature) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
- header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
+ header = HeaderFormat.toLong(preferredBlockSize, replication, isTransient, storagePolicyID);
}
public SnapshotCopy(INodeFile file) {
@@ -75,6 +76,9 @@ public interface INodeFileAttributes extends INodeAttributes {
}
@Override
+ public boolean getLazyPersistFlag() { return HeaderFormat.getLazyPersistFlag(header); }
+
+ @Override
public long getHeaderLong() {
return header;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 512913b..72fc9df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -70,6 +70,8 @@ public class NameNodeLayoutVersion {
"creating file with overwrite"),
XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"),
BLOCK_STORAGE_POLICY(-60, "Block Storage policy");
+ LAZY_PERSIST_FILES(-61, "Support for optional lazy persistence of "
+ + " files with reduced durability guarantees");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index ff33225..b1727c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -221,7 +221,10 @@ public class FSImageFormatPBSnapshot {
.toByteArray(), permission, acl, fileInPb.getModificationTime(),
fileInPb.getAccessTime(), (short) fileInPb.getReplication(),
fileInPb.getPreferredBlockSize(),
- (byte)fileInPb.getStoragePolicyID(), xAttrs);
+ (byte)fileInPb.getStoragePolicyID(),
+ fileInPb.hasIsLazyPersist() ? fileInPb.getIsLazyPersist() : false,
+ xAttrs);
+
}
FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index b68d842..9297c64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -434,6 +434,7 @@ class FSImageLoader {
f.getPermission(), stringTable);
map.put("accessTime", f.getAccessTime());
map.put("blockSize", f.getPreferredBlockSize());
+ map.put("lazyPersist", f.getIsLazyPersist());
map.put("group", p.getGroupName());
map.put("length", getFileSize(f));
map.put("modificationTime", f.getModificationTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 99617b8..744fc75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -247,6 +247,10 @@ public final class PBImageXmlWriter {
.o("perferredBlockSize", f.getPreferredBlockSize())
.o("permission", dumpPermission(f.getPermission()));
+ if (f.hasIsLazyPersist()) {
+ o("lazyPersist", f.getIsLazyPersist());
+ }
+
if (f.getBlocksCount() > 0) {
out.print("<blocks>");
for (BlockProto b : f.getBlocksList()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 588f6c8..85edfab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -139,6 +139,7 @@ message INodeSection {
optional AclFeatureProto acl = 8;
optional XAttrFeatureProto xAttrs = 9;
optional uint32 storagePolicyID = 10;
+ optional bool isLazyPersist = 11 [default = false];
}
message INodeDirectory {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 94b139b..bcce0fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -82,7 +82,7 @@ public class CreateEditsLog {
}
final INodeFile inode = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, blocks, replication, blockSize, (byte)0);
+ p, 0L, 0L, blocks, replication, blockSize, (byte)0, false);
inode.toUnderConstruction("", "");
// Append path to filename with information about blockIDs
@@ -97,7 +97,7 @@ public class CreateEditsLog {
editLog.logMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0, false);
fileUc.toUnderConstruction("", "");
editLog.logOpenFile(filePath, fileUc, false, false);
editLog.logCloseFile(filePath, inode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 89ef215..2cc9be4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -194,7 +194,7 @@ public class TestEditLog {
for (int i = 0; i < numTransactions; i++) {
INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null,
- p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize, (byte)0, false);
inode.toUnderConstruction("", "");
editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
index 9bee4a9..079218a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java
@@ -432,7 +432,7 @@ public class TestFSPermissionChecker {
FsPermission.createImmutable(perm));
INodeFile inodeFile = new INodeFile(INodeId.GRANDFATHER_INODE_ID,
name.getBytes("UTF-8"), permStatus, 0L, 0L, null, REPLICATION,
- PREFERRED_BLOCK_SIZE, (byte)0);
+ PREFERRED_BLOCK_SIZE, (byte)0, false);
parent.addChild(inodeFile);
return inodeFile;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d9d29ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index d688724..bfaf9d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -85,7 +85,7 @@ public class TestINodeFile {
INodeFile createINodeFile(short replication, long preferredBlockSize) {
return new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, replication, preferredBlockSize, (byte)0);
+ null, replication, preferredBlockSize, (byte)0, false);
}
private static INodeFile createINodeFile(byte storagePolicyID) {
@@ -286,7 +286,7 @@ public class TestINodeFile {
INodeFile[] iNodes = new INodeFile[nCount];
for (int i = 0; i < nCount; i++) {
iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
- preferredBlockSize, (byte)0);
+ preferredBlockSize, (byte)0, false);
iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
BlockInfo newblock = new BlockInfo(replication);
iNodes[i].addBlock(newblock);
@@ -344,7 +344,7 @@ public class TestINodeFile {
{//cast from INodeFileUnderConstruction
final INode from = new INodeFile(
INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, replication,
- 1024L, (byte)0);
+ 1024L, (byte)0, false);
from.asFile().toUnderConstruction("client", "machine");
//cast to INodeFile, should success
@@ -1068,7 +1068,7 @@ public class TestINodeFile {
public void testFileUnderConstruction() {
replication = 3;
final INodeFile file = new INodeFile(INodeId.GRANDFATHER_INODE_ID, null,
- perm, 0L, 0L, null, replication, 1024L, (byte)0);
+ perm, 0L, 0L, null, replication, 1024L, (byte)0, false);
assertFalse(file.isUnderConstruction());
final String clientName = "client";
[27/34] git commit: HDFS-6990. Add unit test for evict/delete
RAM_DISK block with open handle. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-6990. Add unit test for evict/delete RAM_DISK block with open handle. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a59ce506
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a59ce506
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a59ce506
Branch: refs/heads/branch-2.6
Commit: a59ce50627357c5aadccc276b5f58b2efa0cf19e
Parents: e9eebaa
Author: arp <ar...@apache.org>
Authored: Mon Sep 22 18:10:28 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../fsdataset/impl/TestScrLazyPersistFiles.java | 341 +++++++++++++++++++
1 file changed, 341 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59ce506/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
new file mode 100644
index 0000000..4270f8c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
@@ -0,0 +1,341 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+ import org.apache.commons.io.IOUtils;
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
+ import org.apache.commons.logging.impl.Log4JLogger;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.CreateFlag;
+ import org.apache.hadoop.fs.FSDataInputStream;
+ import org.apache.hadoop.fs.FSDataOutputStream;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.permission.FsPermission;
+ import org.apache.hadoop.hdfs.*;
+ import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
+ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+ import org.apache.hadoop.hdfs.server.datanode.DataNode;
+ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+ import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
+ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+ import org.apache.hadoop.hdfs.server.namenode.NameNode;
+ import org.apache.hadoop.net.unix.DomainSocket;
+ import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+ import org.apache.hadoop.security.UserGroupInformation;
+ import org.apache.hadoop.test.GenericTestUtils;
+ import org.apache.hadoop.util.NativeCodeLoader;
+ import org.apache.log4j.Level;
+ import org.junit.*;
+
+ import java.io.File;
+ import java.io.IOException;
+ import java.util.Arrays;
+ import java.util.EnumSet;
+ import java.util.List;
+ import java.util.UUID;
+
+ import static org.apache.hadoop.fs.CreateFlag.CREATE;
+ import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+ import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+ import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
+ import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
+ import static org.hamcrest.CoreMatchers.equalTo;
+ import static org.hamcrest.core.Is.is;
+ import static org.junit.Assert.assertThat;
+
+public class TestScrLazyPersistFiles {
+ public static final Log LOG = LogFactory.getLog(TestLazyPersistFiles.class);
+
+ static {
+ ((Log4JLogger) NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) FsDatasetImpl.LOG).getLogger().setLevel(Level.ALL);
+ }
+
+ private static short REPL_FACTOR = 1;
+ private static final int BLOCK_SIZE = 10485760; // 10 MB
+ private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
+ private static final long HEARTBEAT_INTERVAL_SEC = 1;
+ private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
+ private static final int LAZY_WRITER_INTERVAL_SEC = 1;
+ private static final int BUFFER_LENGTH = 4096;
+ private static TemporarySocketDirectory sockDir;
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private DFSClient client;
+ private Configuration conf;
+
+ @BeforeClass
+ public static void init() {
+ sockDir = new TemporarySocketDirectory();
+ DomainSocket.disableBindPathValidation();
+ }
+
+ @AfterClass
+ public static void shutdown() throws IOException {
+ sockDir.close();
+ }
+
+ @Before
+ public void before() {
+ Assume.assumeThat(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS,
+ equalTo(true));
+ Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null));
+ }
+
+ @After
+ public void shutDownCluster() throws IOException {
+ if (fs != null) {
+ fs.close();
+ fs = null;
+ client = null;
+ }
+
+ if (cluster != null) {
+ cluster.shutdownDataNodes();
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ /**
+ * Read in-memory block with Short Circuit Read
+ * Note: the test uses faked RAM_DISK from physical disk.
+ */
+ @Test (timeout=300000)
+ public void testRamDiskShortCircuitRead()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR,
+ new StorageType[]{RAM_DISK, DEFAULT},
+ 2 * BLOCK_SIZE - 1, true); // 1 replica + delta, SCR read
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final int SEED = 0xFADED;
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
+ ensureFileReplicasOnStorageType(path, RAM_DISK);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
+ FSDataInputStream fis = fs.open(path);
+
+ // Verify SCR read counters
+ try {
+ fis = fs.open(path);
+ byte[] buf = new byte[BUFFER_LENGTH];
+ fis.read(0, buf, 0, BUFFER_LENGTH);
+ HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
+ Assert.assertEquals(BUFFER_LENGTH,
+ dfsis.getReadStatistics().getTotalBytesRead());
+ Assert.assertEquals(BUFFER_LENGTH,
+ dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
+ } finally {
+ fis.close();
+ fis = null;
+ }
+ }
+
+ /**
+ * Eviction of lazy persisted blocks with Short Circuit Read handle open
+ * Note: the test uses faked RAM_DISK from physical disk.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000000)
+ public void testRamDiskEvictionWithShortCircuitReadHandle()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ (6 * BLOCK_SIZE -1), true); // 5 replica + delta, SCR.
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+ final int SEED = 0xFADED;
+
+ makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job.
+ // However the block replica should not be evicted from RAM_DISK yet.
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ // No eviction should happen as the free ratio is below the threshold
+ FSDataInputStream fis = fs.open(path1);
+ try {
+ // Keep and open read handle to path1 while creating path2
+ byte[] buf = new byte[BUFFER_LENGTH];
+ fis.read(0, buf, 0, BUFFER_LENGTH);
+
+ // Create the 2nd file that will trigger RAM_DISK eviction.
+ makeTestFile(path2, BLOCK_SIZE * 2, true);
+ ensureFileReplicasOnStorageType(path2, RAM_DISK);
+
+ // Ensure path1 is still readable from the open SCR handle.
+ fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
+ HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
+ Assert.assertEquals(2 * BUFFER_LENGTH,
+ dfsis.getReadStatistics().getTotalBytesRead());
+ Assert.assertEquals(2 * BUFFER_LENGTH,
+ dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
+ } finally {
+ IOUtils.closeQuietly(fis);
+ }
+
+ // After the open handle is closed, path1 should be evicted to DISK.
+ triggerBlockReport();
+ ensureFileReplicasOnStorageType(path1, DEFAULT);
+ }
+
+ // ---- Utility functions for all test cases -------------------------------
+
+ /**
+ * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially
+ * capped. If ramDiskStorageLimit < 0 then it is ignored.
+ */
+ private void startUpCluster(final int numDataNodes,
+ final StorageType[] storageTypes,
+ final long ramDiskStorageLimit,
+ final boolean useSCR)
+ throws IOException {
+
+ conf = new Configuration();
+ conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC,
+ LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
+ conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
+ conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ HEARTBEAT_RECHECK_INTERVAL_MSEC);
+ conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
+ LAZY_WRITER_INTERVAL_SEC);
+
+ if (useSCR)
+ {
+ conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY,useSCR);
+ conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT,
+ UUID.randomUUID().toString());
+ conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
+ new File(sockDir.getDir(),
+ "TestShortCircuitLocalReadHandle._PORT.sock").getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ }
+
+ REPL_FACTOR = 1; //Reset in case a test has modified the value
+
+ cluster = new MiniDFSCluster
+ .Builder(conf)
+ .numDataNodes(numDataNodes)
+ .storageTypes(storageTypes != null ? storageTypes : new StorageType[] { DEFAULT, DEFAULT })
+ .build();
+ fs = cluster.getFileSystem();
+ client = fs.getClient();
+
+ // Artificially cap the storage capacity of the RAM_DISK volume.
+ if (ramDiskStorageLimit >= 0) {
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+
+ for (FsVolumeSpi volume : volumes) {
+ if (volume.getStorageType() == RAM_DISK) {
+ ((FsTransientVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
+ }
+ }
+ }
+
+ LOG.info("Cluster startup complete");
+ }
+
+ private void startUpCluster(final int numDataNodes,
+ final StorageType[] storageTypes,
+ final long ramDiskStorageLimit)
+ throws IOException {
+ startUpCluster(numDataNodes, storageTypes, ramDiskStorageLimit, false);
+ }
+
+ private void makeTestFile(Path path, long length, final boolean isLazyPersist)
+ throws IOException {
+
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+
+ if (isLazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+
+ FSDataOutputStream fos = null;
+ try {
+ fos =
+ fs.create(path,
+ FsPermission.getFileDefault(),
+ createFlags,
+ BUFFER_LENGTH,
+ REPL_FACTOR,
+ BLOCK_SIZE,
+ null);
+
+ // Allocate a block.
+ byte[] buffer = new byte[BUFFER_LENGTH];
+ for (int bytesWritten = 0; bytesWritten < length; ) {
+ fos.write(buffer, 0, buffer.length);
+ bytesWritten += buffer.length;
+ }
+ if (length > 0) {
+ fos.hsync();
+ }
+ } finally {
+ IOUtils.closeQuietly(fos);
+ }
+ }
+
+ private LocatedBlocks ensureFileReplicasOnStorageType(
+ Path path, StorageType storageType) throws IOException {
+ // Ensure that returned block locations returned are correct!
+ LOG.info("Ensure path: " + path + " is on StorageType: " + storageType);
+ assertThat(fs.exists(path), is(true));
+ long fileLength = client.getFileInfo(path.toString()).getLen();
+ LocatedBlocks locatedBlocks =
+ client.getLocatedBlocks(path.toString(), 0, fileLength);
+ for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+ assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
+ }
+ return locatedBlocks;
+ }
+
+ private void makeRandomTestFile(Path path, long length, final boolean isLazyPersist,
+ long seed) throws IOException {
+ DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
+ BLOCK_SIZE, REPL_FACTOR, seed, true);
+ }
+
+ private boolean verifyReadRandomFile(
+ Path path, int fileLength, int seed) throws IOException {
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, path);
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(seed, fileLength);
+ return Arrays.equals(contents, expected);
+ }
+
+ private void triggerBlockReport()
+ throws IOException, InterruptedException {
+ // Trigger block report to NN
+ DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
+ Thread.sleep(10 * 1000);
+ }
+}
[20/34] git commit: HDFS-7064. Fix unit test failures in HDFS-6581
branch. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-7064. Fix unit test failures in HDFS-6581 branch. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3ccbd8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3ccbd8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3ccbd8e
Branch: refs/heads/branch-2.6
Commit: d3ccbd8e65d69c00917e78fc8398bf1605eaa137
Parents: 6cc206b
Author: arp <ar...@apache.org>
Authored: Tue Sep 16 15:45:07 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/CommandWithDestination.java | 3 ++-
.../hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java | 5 +++--
.../org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java | 2 +-
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3ccbd8e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 5d0d9d6..7152f02 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -478,7 +478,8 @@ abstract class CommandWithDestination extends FsCommand {
createFlags.add(LAZY_PERSIST);
}
return create(item.path,
- null,
+ FsPermission.getFileDefault().applyUMask(
+ FsPermission.getUMask(getConf())),
createFlags,
getConf().getInt("io.file.buffer.size", 4096),
lazyPersist ? 1 : getDefaultReplication(item.path),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3ccbd8e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 217e932..e2efb2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -381,8 +381,9 @@ public class FsVolumeImpl implements FsVolumeSpi {
!FileUtil.fullyDelete(finalizedDir)) {
throw new IOException("Failed to delete " + finalizedDir);
}
- if (!DatanodeUtil.dirNoFilesRecursive(lazypersistDir) ||
- !FileUtil.fullyDelete(lazypersistDir)) {
+ if (lazypersistDir.exists() &&
+ ((!DatanodeUtil.dirNoFilesRecursive(lazypersistDir) ||
+ !FileUtil.fullyDelete(lazypersistDir)))) {
throw new IOException("Failed to delete " + lazypersistDir);
}
FileUtil.fullyDelete(tmpDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3ccbd8e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
index c0b4f9a..94af015 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
@@ -51,7 +51,7 @@ public class TestDataDirs {
String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3,[ram_disk]/dir4";
conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1);
locations = DataNode.getStorageLocations(conf);
- assertThat(locations.size(), is(4));
+ assertThat(locations.size(), is(5));
assertThat(locations.get(0).getStorageType(), is(StorageType.DISK));
assertThat(locations.get(0).getUri(), is(dir0.toURI()));
assertThat(locations.get(1).getStorageType(), is(StorageType.DISK));
[13/34] git commit: HDFS-6977. Delete all copies when a block is
deleted from the block space. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6977. Delete all copies when a block is deleted from the block space. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/186d1950
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/186d1950
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/186d1950
Branch: refs/heads/branch-2.6
Commit: 186d19503371ab71d0a6c0bb85df507eecc2c247
Parents: 34f28ea
Author: arp <ar...@apache.org>
Authored: Mon Sep 8 10:35:54 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/BlockPoolSlice.java | 13 +++--
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +++---
.../fsdataset/impl/LazyWriteReplicaTracker.java | 51 ++++++++++++++------
.../fsdataset/impl/TestLazyPersistFiles.java | 4 +-
4 files changed, 53 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/186d1950/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 637705a..f5c6ed6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -270,17 +270,16 @@ class BlockPoolSlice {
* Save the given replica to persistent storage.
*
* @param replicaInfo
- * @return The saved block file.
+ * @return The saved meta and block files, in that order.
* @throws IOException
*/
- File lazyPersistReplica(ReplicaInfo replicaInfo) throws IOException {
+ File[] lazyPersistReplica(ReplicaInfo replicaInfo) throws IOException {
if (!lazypersistDir.exists() && !lazypersistDir.mkdirs()) {
FsDatasetImpl.LOG.warn("Failed to create " + lazypersistDir);
}
- File metaFile = FsDatasetImpl.copyBlockFiles(replicaInfo, lazypersistDir);
- File blockFile = Block.metaToBlockFile(metaFile);
- dfsUsage.incDfsUsed(replicaInfo.getNumBytes() + metaFile.length());
- return blockFile;
+ File targetFiles[] = FsDatasetImpl.copyBlockFiles(replicaInfo, lazypersistDir);
+ dfsUsage.incDfsUsed(targetFiles[0].length() + targetFiles[1].length());
+ return targetFiles;
}
/**
@@ -484,7 +483,7 @@ class BlockPoolSlice {
if (newReplica.getVolume().isTransientStorage()) {
lazyWriteReplicaMap.addReplica(bpid, blockId, newReplica.getVolume());
} else {
- lazyWriteReplicaMap.discardReplica(bpid, blockId, true);
+ lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/186d1950/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 8d93dcc..19260db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -707,10 +707,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
/**
* Copy the block and meta files for the given block from the given
- * @return the new meta file.
+ * @return the new meta and block files.
* @throws IOException
*/
- static File copyBlockFiles(ReplicaInfo replicaInfo, File destRoot)
+ static File[] copyBlockFiles(ReplicaInfo replicaInfo, File destRoot)
throws IOException {
final File destDir = DatanodeUtil.idToBlockDir(destRoot, replicaInfo.getBlockId());
final File dstFile = new File(destDir, replicaInfo.getBlockName());
@@ -731,7 +731,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.debug("addBlock: Moved " + srcMeta + " to " + dstMeta);
LOG.debug("addBlock: Moved " + srcFile + " to " + dstFile);
}
- return dstMeta;
+ return new File[] {dstMeta, dstFile};
}
static private void truncateBlock(File blockFile, File metaFile,
@@ -2332,13 +2332,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
- File savedBlockFile = targetVolume.getBlockPoolSlice(bpid)
- .lazyPersistReplica(replicaInfo);
- lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedBlockFile);
+ File[] savedFiles = targetVolume.getBlockPoolSlice(bpid)
+ .lazyPersistReplica(replicaInfo);
+ lazyWriteReplicaTracker.recordEndLazyPersist(
+ bpid, blockId, savedFiles[0], savedFiles[1]);
if (LOG.isDebugEnabled()) {
LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
- " to file " + savedBlockFile);
+ " to file " + savedFiles[1]);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/186d1950/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
index 9f020c4..a8ab1b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
@@ -48,6 +48,7 @@ class LazyWriteReplicaTracker {
* Persistent volume that holds or will hold the saved replica.
*/
FsVolumeImpl lazyPersistVolume;
+ File savedMetaFile;
File savedBlockFile;
ReplicaState(final String bpid, final long blockId, FsVolumeSpi transientVolume) {
@@ -56,9 +57,26 @@ class LazyWriteReplicaTracker {
this.transientVolume = transientVolume;
state = State.IN_MEMORY;
lazyPersistVolume = null;
+ savedMetaFile = null;
savedBlockFile = null;
}
+ void deleteSavedFiles() {
+ try {
+ if (savedBlockFile != null) {
+ savedBlockFile.delete();
+ savedBlockFile = null;
+ }
+
+ if (savedMetaFile != null) {
+ savedMetaFile.delete();
+ savedMetaFile = null;
+ }
+ } catch (Throwable t) {
+ // Ignore any exceptions.
+ }
+ }
+
@Override
public String toString() {
return "[Bpid=" + bpid + ";blockId=" + blockId + "]";
@@ -144,7 +162,8 @@ class LazyWriteReplicaTracker {
}
synchronized void recordEndLazyPersist(
- final String bpid, final long blockId, File savedBlockFile) {
+ final String bpid, final long blockId,
+ final File savedMetaFile, final File savedBlockFile) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
ReplicaState replicaState = map.get(blockId);
@@ -153,6 +172,7 @@ class LazyWriteReplicaTracker {
bpid + "; blockId=" + blockId);
}
replicaState.state = State.LAZY_PERSIST_COMPLETE;
+ replicaState.savedMetaFile = savedMetaFile;
replicaState.savedBlockFile = savedBlockFile;
if (replicasNotPersisted.peek() == replicaState) {
@@ -208,12 +228,22 @@ class LazyWriteReplicaTracker {
return null;
}
- void discardReplica(ReplicaState replicaState, boolean force) {
- discardReplica(replicaState.bpid, replicaState.blockId, force);
+ void discardReplica(ReplicaState replicaState, boolean deleteSavedCopies) {
+ discardReplica(replicaState.bpid, replicaState.blockId, deleteSavedCopies);
}
+ /**
+ * Discard any state we are tracking for the given replica. This could mean
+ * the block is either deleted from the block space or the replica is no longer
+ * on transient storage.
+ *
+ * @param deleteSavedCopies true if we should delete the saved copies on
+ * persistent storage. This should be set by the
+ * caller when the block is no longer needed.
+ */
synchronized void discardReplica(
- final String bpid, final long blockId, boolean force) {
+ final String bpid, final long blockId,
+ boolean deleteSavedCopies) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
if (map == null) {
@@ -223,19 +253,12 @@ class LazyWriteReplicaTracker {
ReplicaState replicaState = map.get(blockId);
if (replicaState == null) {
- if (force) {
- return;
- }
- throw new IllegalStateException("Unknown replica bpid=" +
- bpid + "; blockId=" + blockId);
+ return;
}
- if (replicaState.state != State.LAZY_PERSIST_COMPLETE && !force) {
- throw new IllegalStateException("Discarding replica without " +
- "saving it to disk bpid=" + bpid + "; blockId=" + blockId);
-
+ if (deleteSavedCopies) {
+ replicaState.deleteSavedFiles();
}
-
map.remove(blockId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/186d1950/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 461c44d..7dfba50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -394,7 +394,7 @@ public class TestLazyPersistFiles {
public void testRamDiskEviction()
throws IOException, InterruptedException {
startUpCluster(REPL_FACTOR,
- new StorageType[] {RAM_DISK, DEFAULT },
+ new StorageType[] { RAM_DISK, DEFAULT },
(2 * BLOCK_SIZE - 1)); // 1 replica + delta.
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -415,7 +415,7 @@ public class TestLazyPersistFiles {
// Make sure that the second file's block replica is on RAM_DISK, whereas
// the original file's block replica is now on disk.
-// ensureFileReplicasOnStorageType(path2, RAM_DISK);
+ ensureFileReplicasOnStorageType(path2, RAM_DISK);
ensureFileReplicasOnStorageType(path1, DEFAULT);
}
[28/34] git commit: HDFS-6932. Balancer and Mover tools should ignore
replicas on RAM_DISK. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de4d189
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de4d189
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de4d189
Branch: refs/heads/branch-2.6
Commit: 9de4d1897a014705e8846fe6d1d71b6b5054b9fd
Parents: a59ce50
Author: arp <ar...@apache.org>
Authored: Wed Sep 24 21:08:59 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/StorageType.java | 31 +++++--
.../hadoop/hdfs/server/balancer/Balancer.java | 2 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 30 ++++--
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 33 +++++++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +-
.../hdfs/server/balancer/TestBalancer.java | 96 +++++++++++++++++--
.../hdfs/server/mover/TestStorageMover.java | 97 +++++++++++++++++++-
7 files changed, 265 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index c11d4e6..99cae9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs;
import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,18 +32,34 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public enum StorageType {
- DISK,
- SSD,
- ARCHIVE,
- RAM_DISK;
+ DISK(false),
+ SSD(false),
+ ARCHIVE(false),
+ RAM_DISK(true);
+
+ private final boolean isTransient;
public static final StorageType DEFAULT = DISK;
-
+
public static final StorageType[] EMPTY_ARRAY = {};
-
+
private static final StorageType[] VALUES = values();
-
+
+ StorageType(boolean isTransient) { this.isTransient = isTransient; }
+
+ public boolean isMovable() { return isTransient == false; }
+
public static List<StorageType> asList() {
return Arrays.asList(VALUES);
}
+
+ public static List<StorageType> getMovableTypes() {
+ List<StorageType> movableTypes = new ArrayList<StorageType>();
+ for (StorageType t : VALUES) {
+ if ( t.isTransient == false ) {
+ movableTypes.add(t);
+ }
+ }
+ return movableTypes;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index d77da7c..87348b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -272,7 +272,7 @@ public class Balancer {
long overLoadedBytes = 0L, underLoadedBytes = 0L;
for(DatanodeStorageReport r : reports) {
final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
- for(StorageType t : StorageType.asList()) {
+ for(StorageType t : StorageType.getMovableTypes()) {
final Double utilization = policy.getUtilization(r, t);
if (utilization == null) { // datanode does not have such storage type
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 04133bd..59814af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -69,7 +69,7 @@ public class Mover {
= new EnumMap<StorageType, List<StorageGroup>>(StorageType.class);
private StorageMap() {
- for(StorageType t : StorageType.asList()) {
+ for(StorageType t : StorageType.getMovableTypes()) {
targetStorageTypeMap.put(t, new LinkedList<StorageGroup>());
}
}
@@ -130,7 +130,7 @@ public class Mover {
final List<DatanodeStorageReport> reports = dispatcher.init();
for(DatanodeStorageReport r : reports) {
final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
- for(StorageType t : StorageType.asList()) {
+ for(StorageType t : StorageType.getMovableTypes()) {
final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher);
final long maxRemaining = getMaxRemaining(r, t);
final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t,
@@ -348,7 +348,7 @@ public class Mover {
LocatedBlock lb = lbs.get(i);
final StorageTypeDiff diff = new StorageTypeDiff(types,
lb.getStorageTypes());
- if (!diff.removeOverlap()) {
+ if (!diff.removeOverlap(true)) {
if (scheduleMoves4Block(diff, lb)) {
hasRemaining |= (diff.existing.size() > 1 &&
diff.expected.size() > 1);
@@ -452,22 +452,38 @@ public class Mover {
this.expected = new LinkedList<StorageType>(expected);
this.existing = new LinkedList<StorageType>(Arrays.asList(existing));
}
-
+
/**
* Remove the overlap between the expected types and the existing types.
- * @return if the existing types or the expected types is empty after
+ * @param ignoreNonMovable ignore non-movable storage types
+ * by removing them from both expected and existing storage type list
+ * to prevent non-movable storage from being moved.
+ * @returns if the existing types or the expected types is empty after
* removing the overlap.
*/
- boolean removeOverlap() {
+ boolean removeOverlap(boolean ignoreNonMovable) {
for(Iterator<StorageType> i = existing.iterator(); i.hasNext(); ) {
final StorageType t = i.next();
if (expected.remove(t)) {
i.remove();
}
}
+ if (ignoreNonMovable) {
+ removeNonMovable(existing);
+ removeNonMovable(expected);
+ }
return expected.isEmpty() || existing.isEmpty();
}
-
+
+ void removeNonMovable(List<StorageType> types) {
+ for (Iterator<StorageType> i = types.iterator(); i.hasNext(); ) {
+ final StorageType t = i.next();
+ if (!t.isMovable()) {
+ i.remove();
+ }
+ }
+ }
+
@Override
public String toString() {
return getClass().getSimpleName() + "{expected=" + expected
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 820b812..c728b2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1457,6 +1457,39 @@ public class DFSTestUtil {
}
/**
+ * Helper function that verified blocks of a file are placed on the
+ * expected storage type.
+ *
+ * @param fs The file system containing the the file.
+ * @param client The DFS client used to access the file
+ * @param path name to the file to verify
+ * @param storageType expected storage type
+ * @returns true if file exists and its blocks are located on the expected
+ * storage type.
+ * false otherwise.
+ */
+ public static boolean verifyFileReplicasOnStorageType(FileSystem fs,
+ DFSClient client, Path path, StorageType storageType) throws IOException {
+ if (!fs.exists(path)) {
+ LOG.info("verifyFileReplicasOnStorageType: file " + path + "does not exist");
+ return false;
+ }
+ long fileLength = client.getFileInfo(path.toString()).getLen();
+ LocatedBlocks locatedBlocks =
+ client.getLocatedBlocks(path.toString(), 0, fileLength);
+ for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+ if (locatedBlock.getStorageTypes()[0] != storageType) {
+ LOG.info("verifyFileReplicasOnStorageType: for file " + path +
+ ". Expect blk" + locatedBlock +
+ " on Type: " + storageType + ". Actual Type: " +
+ locatedBlock.getStorageTypes()[0]);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
* Helper function to create a key in the Key Provider. Defaults
* to the first indexed NameNode's Key Provider.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 7034715..bd8e390 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1394,7 +1394,8 @@ public class MiniDFSCluster {
// Set up datanode address
setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
if (manageDfsDirs) {
- String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]);
+ String dirs = makeDataNodeDirs(i, storageTypes == null ?
+ null : storageTypes[i - curDatanodesNum]);
dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs);
conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 1df31ac..831e2c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.balancer;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
+import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -41,12 +44,7 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.*;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -57,6 +55,7 @@ import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Parameters;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.log4j.Level;
@@ -86,6 +85,7 @@ public class TestBalancer {
static final double CAPACITY_ALLOWED_VARIANCE = 0.005; // 0.5%
static final double BALANCE_ALLOWED_VARIANCE = 0.11; // 10%+delta
static final int DEFAULT_BLOCK_SIZE = 100;
+ static final int DEFAULT_RAM_DISK_BLOCK_SIZE = 5 * 1024 * 1024;
private static final Random r = new Random();
static {
@@ -108,6 +108,15 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
}
+ static void initConfWithRamDisk(Configuration conf) {
+ conf.setLong(DFS_BLOCK_SIZE_KEY, DEFAULT_RAM_DISK_BLOCK_SIZE);
+ conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, 3);
+ conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
+ conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1);
+ conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS, 1);
+ }
+
/* create a file with a length of <code>fileLen</code> */
static void createFile(MiniDFSCluster cluster, Path filePath, long fileLen,
short replicationFactor, int nnIndex)
@@ -1098,6 +1107,81 @@ public class TestBalancer {
CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, true);
}
+ /*
+ * Test Balancer with Ram_Disk configured
+ * One DN has two files on RAM_DISK, other DN has no files on RAM_DISK.
+ * Then verify that the balancer does not migrate files on RAM_DISK across DN.
+ */
+ @Test(timeout=300000)
+ public void testBalancerWithRamDisk() throws Exception {
+ final int SEED = 0xFADED;
+ final short REPL_FACT = 1;
+ Configuration conf = new Configuration();
+ initConfWithRamDisk(conf);
+
+ final int defaultRamDiskCapacity = 10;
+ final int defaultDiskCapacity = 100;
+ final long ramDiskStorageLimit =
+ ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
+ (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
+ final long diskStorageLimit =
+ ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
+ (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
+
+ cluster = new MiniDFSCluster
+ .Builder(conf)
+ .numDataNodes(1)
+ .storageCapacities(new long[] { ramDiskStorageLimit, diskStorageLimit })
+ .storageTypes(new StorageType[] { RAM_DISK, DEFAULT })
+ .build();
+
+ try {
+ cluster.waitActive();
+ // Create few files on RAM_DISK
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ final Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+
+ DistributedFileSystem fs = cluster.getFileSystem();
+ DFSClient client = fs.getClient();
+ DFSTestUtil.createFile(fs, path1, true,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, 4 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
+ DFSTestUtil.createFile(fs, path1, true,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, 1 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(6 * 1000);
+
+ // Add another fresh DN with the same type/capacity without files on RAM_DISK
+ StorageType[][] storageTypes = new StorageType[][] {{RAM_DISK, DEFAULT}};
+ long[][] storageCapacities = new long[][]{{ramDiskStorageLimit, diskStorageLimit}};
+ cluster.startDataNodes(conf, REPL_FACT, storageTypes, true, null,
+ null, null, storageCapacities, null, false, false, false, null);
+
+ cluster.triggerHeartbeats();
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+
+ // Run Balancer
+ Balancer.Parameters p = new Balancer.Parameters(
+ Parameters.DEFAULT.policy,
+ Parameters.DEFAULT.threshold,
+ Parameters.DEFAULT.nodesToBeExcluded,
+ Parameters.DEFAULT.nodesToBeIncluded);
+ final int r = Balancer.run(namenodes, p, conf);
+
+ // Validate no RAM_DISK block should be moved
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+
+ // Verify files are still on RAM_DISK
+ DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path1, RAM_DISK);
+ DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path2, RAM_DISK);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* @param args
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de4d189/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 2484c43..b690165 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -67,6 +68,8 @@ import org.junit.Test;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC;
+
/**
* Test the data migration tool (for Archival Storage)
*/
@@ -326,10 +329,10 @@ public class TestStorageMover {
Assert.assertTrue(fileStatus.getFullName(parent.toString())
+ " with policy " + policy + " has non-empty overlap: " + diff
+ ", the corresponding block is " + lb.getBlock().getLocalBlock(),
- diff.removeOverlap());
+ diff.removeOverlap(true));
}
}
-
+
Replication getReplication(Path file) throws IOException {
return getOrVerifyReplication(file, null);
}
@@ -397,17 +400,29 @@ public class TestStorageMover {
}
private static StorageType[][] genStorageTypes(int numDataNodes) {
- return genStorageTypes(numDataNodes, 0, 0);
+ return genStorageTypes(numDataNodes, 0, 0, 0);
}
private static StorageType[][] genStorageTypes(int numDataNodes,
int numAllDisk, int numAllArchive) {
+ return genStorageTypes(numDataNodes, numAllDisk, numAllArchive, 0);
+ }
+
+ private static StorageType[][] genStorageTypes(int numDataNodes,
+ int numAllDisk, int numAllArchive, int numRamDisk) {
+ Preconditions.checkArgument(
+ (numAllDisk + numAllArchive + numRamDisk) <= numDataNodes);
+
StorageType[][] types = new StorageType[numDataNodes][];
int i = 0;
- for (; i < numAllDisk; i++) {
+ for (; i < numRamDisk; i++)
+ {
+ types[i] = new StorageType[]{StorageType.RAM_DISK, StorageType.DISK};
+ }
+ for (; i < numRamDisk + numAllDisk; i++) {
types[i] = new StorageType[]{StorageType.DISK, StorageType.DISK};
}
- for (; i < numAllDisk + numAllArchive; i++) {
+ for (; i < numRamDisk + numAllDisk + numAllArchive; i++) {
types[i] = new StorageType[]{StorageType.ARCHIVE, StorageType.ARCHIVE};
}
for (; i < types.length; i++) {
@@ -416,6 +431,26 @@ public class TestStorageMover {
return types;
}
+ private static long[][] genCapacities(int nDatanodes, int numAllDisk,
+ int numAllArchive, int numRamDisk, long diskCapacity,
+ long archiveCapacity, long ramDiskCapacity) {
+ final long[][] capacities = new long[nDatanodes][];
+ int i = 0;
+ for (; i < numRamDisk; i++) {
+ capacities[i] = new long[]{ramDiskCapacity, diskCapacity};
+ }
+ for (; i < numRamDisk + numAllDisk; i++) {
+ capacities[i] = new long[]{diskCapacity, diskCapacity};
+ }
+ for (; i < numRamDisk + numAllDisk + numAllArchive; i++) {
+ capacities[i] = new long[]{archiveCapacity, archiveCapacity};
+ }
+ for(; i < capacities.length; i++) {
+ capacities[i] = new long[]{diskCapacity, archiveCapacity};
+ }
+ return capacities;
+ }
+
private static class PathPolicyMap {
final Map<Path, BlockStoragePolicy> map = Maps.newHashMap();
final Path hot = new Path("/hot");
@@ -748,4 +783,56 @@ public class TestStorageMover {
test.shutdownCluster();
}
}
+
+ /**
+ * Test blocks of lazy_persist file on RAM_DISK will not be moved to other
+ * storage types by the Storage Mover.
+ */
+ @Test
+ public void testRamDiskNotMoved() throws Exception {
+ LOG.info("testRamDiskNotMoved");
+ final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
+ final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
+
+ final long diskCapacity = 100 * BLOCK_SIZE;
+ final long archiveCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
+ * BLOCK_SIZE;
+ final long ramDiskCapacity = 10 * BLOCK_SIZE;
+ final long[][] capacities = genCapacities(1, 0, 0, 1,
+ diskCapacity, archiveCapacity, ramDiskCapacity);
+ final int LAZY_WRITER_INTERVAL_SEC = 1;
+ final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
+ 1, (short)1, genStorageTypes(1, 0, 0, 1), capacities);
+ clusterScheme.conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
+ LAZY_WRITER_INTERVAL_SEC);
+ final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
+
+ try {
+ test.runBasicTest(false);
+
+ // test creating a hot RAM_DISK file
+ final int SEED = 0xFADED;
+ final Path foo_hot = new Path(pathPolicyMap.hot, "foo_hot");
+ DFSTestUtil.createFile(test.dfs, foo_hot, true, BLOCK_SIZE, BLOCK_SIZE,
+ BLOCK_SIZE, (short) 1, SEED, true);
+ Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
+ test.dfs.getClient(), foo_hot, StorageType.RAM_DISK));
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ // Verify policy related name change is allowed
+ final Path foo_hot_new = new Path(pathPolicyMap.warm, "foo_hot");
+ test.dfs.rename(foo_hot, pathPolicyMap.warm);
+ Assert.assertTrue(test.dfs.exists(foo_hot_new));
+
+ // Verify blocks on ram disk will not be moved to other storage types by
+ // policy based Storage Mover.
+ test.migrate();
+ Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
+ test.dfs.getClient(), foo_hot_new, StorageType.RAM_DISK));
+ } finally {
+ test.shutdownCluster();
+ }
+ }
}
[33/34] git commit: HDFS-7112. LazyWriter should use either async IO
or one thread per physical disk. Contributed by Xiaoyu Yao.
Posted by ji...@apache.org.
HDFS-7112. LazyWriter should use either async IO or one thread per physical disk. Contributed by Xiaoyu Yao.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b6d115c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b6d115c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b6d115c
Branch: refs/heads/branch-2.6
Commit: 1b6d115ccf9b8eef1989a3aca73c1ba3165c95f6
Parents: 137f0b8
Author: cnauroth <cn...@apache.org>
Authored: Tue Oct 7 20:25:19 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:53 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 95 +++++++
.../server/datanode/fsdataset/FsDatasetSpi.java | 13 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 21 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 179 ++++++++-----
.../datanode/fsdataset/impl/FsVolumeImpl.java | 15 +-
.../impl/RamDiskAsyncLazyPersistService.java | 252 +++++++++++++++++++
.../server/datanode/SimulatedFSDataset.java | 12 +
.../fsdataset/impl/TestLazyPersistFiles.java | 4 +-
8 files changed, 506 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8f4d6ee..afb8d35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -570,6 +570,101 @@ Release 2.6.0 - UNRELEASED
HDFS-5089. When a LayoutVersion support SNAPSHOT, it must support
FSIMAGE_NAME_OPTIMIZATION. (szetszwo)
+ BREAKDOWN OF HDFS-6581 SUBTASKS AND RELATED JIRAS
+
+ HDFS-6921. Add LazyPersist flag to FileStatus. (Arpit Agarwal)
+
+ HDFS-6924. Add new RAM_DISK storage type. (Arpit Agarwal)
+
+ HDFS-6922. Add LazyPersist flag to INodeFile, save it in FsImage and
+ edit logs. (Arpit Agarwal)
+
+ HDFS-6923. Propagate LazyPersist flag to DNs via DataTransferProtocol.
+ (Arpit Agarwal)
+
+ HDFS-6925. DataNode should attempt to place replicas on transient storage
+ first if lazyPersist flag is received. (Arpit Agarwal)
+
+ HDFS-6926. DN support for saving replicas to persistent storage and
+ evicting in-memory replicas. (Arpit Agarwal)
+
+ HDFS-6927. Initial unit tests for lazy persist files. (Arpit Agarwal)
+
+ HDFS-6929. NN periodically unlinks lazy persist files with missing
+ replicas from namespace. (Arpit Agarwal)
+
+ HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing.
+ (Arpit Agarwal)
+
+ HDFS-6960. Bugfix in LazyWriter, fix test case and some refactoring.
+ (Arpit Agarwal)
+
+ HDFS-6931. Move lazily persisted replicas to finalized directory on DN
+ startup. (Arpit Agarwal)
+
+ HDFS-6950. Add Additional unit tests for HDFS-6581. (Xiaoyu Yao via
+ Arpit Agarwal)
+
+ HDFS-6930. Improve replica eviction from RAM disk. (Arpit Agarwal)
+
+ HDFS-6977. Delete all copies when a block is deleted from the block space.
+ (Arpit Agarwal)
+
+ HDFS-6991. Notify NN of evicted block before deleting it from RAM disk.
+ (Arpit Agarwal)
+
+ HDFS-6978. Directory scanner should correctly reconcile blocks on RAM
+ disk. (Arpit Agarwal)
+
+ HDFS-7066. LazyWriter#evictBlocks misses a null check for replicaState.
+ (Xiaoyu Yao via Arpit Agarwal)
+
+ HDFS-7064. Fix unit test failures in HDFS-6581 branch. (Xiaoyu Yao via
+ Arpit Agarwal)
+
+ HDFS-6581. Few more unit test fixes for HDFS-6581. (Arpit Agarwal)
+
+ HDFS-7080. Fix finalize and upgrade unit test failures. (Arpit Agarwal)
+
+ HDFS-7084. FsDatasetImpl#copyBlockFiles debug log can be improved.
+ (Xiaoyu Yao via Arpit Agarwal)
+
+ HDFS-7091. Add forwarding constructor for INodeFile for existing callers.
+ (Arpit Agarwal)
+
+ HDFS-7100. Make eviction scheme pluggable. (Arpit Agarwal)
+
+ HDFS-7108. Fix unit test failures in SimulatedFsDataset. (Arpit Agarwal)
+
+ HDFS-6990. Add unit test for evict/delete RAM_DISK block with open
+ handle. (Xiaoyu Yao via Arpit Agarwal)
+
+ HDFS-7143. Fix findbugs warnings in HDFS-6581 branch. (szetszwo via
+ Arpit Agarwal)
+
+ HDFS-6932. Balancer and Mover tools should ignore replicas on RAM_DISK.
+ (Xiaoyu Yao via Arpit Agarwal)
+
+ HDFS-7144. Fix findbugs warnings in RamDiskReplicaTracker. (szetszwo via
+ Arpit Agarwal)
+
+ HDFS-7155. Bugfix in createLocatedFileStatus caused by bad merge.
+ (Arpit Agarwal)
+
+ HDFS-7153. Add storagePolicy to NN edit log during file creation.
+ (Arpit Agarwal)
+
+ HDFS-7159. Use block storage policy to set lazy persist preference.
+ (Arpit Agarwal)
+
+ HDFS-7129. Metrics to track usage of memory for writes. (Xiaoyu Yao
+ via Arpit Agarwal)
+
+ HDFS-7171. Fix Jenkins failures in HDFS-6581 branch. (Arpit Agarwal)
+
+ HDFS-7112. LazyWriter should use either async IO or one thread per physical
+ disk. (Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 2bb2e7f..3f1400d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.datanode.Replica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -463,5 +464,15 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
public void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
final FileDescriptor fd, final long offset, final long nbytes,
final int flags);
-}
+ /**
+ * Callback from RamDiskAsyncLazyPersistService upon async lazy persist task end
+ */
+ public void onCompleteLazyPersist(String bpId, long blockId,
+ long creationTime, File[] savedFiles, FsVolumeImpl targetVolume);
+
+ /**
+ * Callback from RamDiskAsyncLazyPersistService upon async lazy persist task fail
+ */
+ public void onFailLazyPersist(String bpId, long blockId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 3f58d38..dce2ff8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -171,6 +171,10 @@ class BlockPoolSlice {
long getDfsUsed() throws IOException {
return dfsUsage.getUsed();
}
+
+ void incDfsUsed(long value) {
+ dfsUsage.incDfsUsed(value);
+ }
/**
* Read in the cached DU value and return it if it is less than 600 seconds
@@ -277,23 +281,6 @@ class BlockPoolSlice {
}
/**
- * Save the given replica to persistent storage.
- *
- * @return The saved meta and block files, in that order.
- * @throws IOException
- */
- File[] lazyPersistReplica(long blockId, long genStamp,
- File srcMeta, File srcFile) throws IOException {
- if (!lazypersistDir.exists() && !lazypersistDir.mkdirs()) {
- FsDatasetImpl.LOG.warn("Failed to create " + lazypersistDir);
- }
- File targetFiles[] = FsDatasetImpl.copyBlockFiles(
- blockId, genStamp, srcMeta, srcFile, lazypersistDir);
- dfsUsage.incDfsUsed(targetFiles[0].length() + targetFiles[1].length());
- return targetFiles;
- }
-
- /**
* Move a persisted replica from lazypersist directory to a subdirectory
* under finalized.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index f2daf99..07e19cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -220,6 +220,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final ReplicaMap volumeMap;
final RamDiskReplicaTracker ramDiskReplicaTracker;
+ final RamDiskAsyncLazyPersistService asyncLazyPersistService;
private static final int MAX_BLOCK_EVICTIONS_PER_ITERATION = 3;
@@ -273,10 +274,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
VolumeChoosingPolicy.class), conf);
volumes = new FsVolumeList(volsFailed, blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode);
+ asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode);
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
addVolume(dataLocations, storage.getStorageDir(idx));
}
+ setupAsyncLazyPersistThreads();
cacheManager = new FsDatasetCache(this);
@@ -409,6 +412,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
+ setupAsyncLazyPersistThreads();
+
for (int i = 0; i < volumes.size(); i++) {
if (successFlags[i]) {
succeedVolumes.add(volumes.get(i));
@@ -462,6 +467,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
storageMap.remove(sd.getStorageUuid());
}
}
+ setupAsyncLazyPersistThreads();
}
private StorageType getStorageTypeFromLocations(
@@ -1506,10 +1512,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
RamDiskReplica replicaInfo =
ramDiskReplicaTracker.getReplica(bpid, invalidBlks[i].getBlockId());
if (replicaInfo != null) {
- if (replicaInfo.getIsPersisted() == false) {
+ if (!replicaInfo.getIsPersisted()) {
datanode.getMetrics().incrRamDiskBlocksDeletedBeforeLazyPersisted();
}
- discardRamDiskReplica(replicaInfo, true);
+ ramDiskReplicaTracker.discardReplica(replicaInfo.getBlockPoolId(),
+ replicaInfo.getBlockId(), true);
}
}
@@ -1750,6 +1757,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (asyncDiskService != null) {
asyncDiskService.shutdown();
}
+
+ if (asyncLazyPersistService != null) {
+ asyncLazyPersistService.shutdown();
+ }
if(volumes != null) {
volumes.shutdown();
@@ -2309,6 +2320,40 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override
+ public void onCompleteLazyPersist(String bpId, long blockId,
+ long creationTime, File[] savedFiles, FsVolumeImpl targetVolume) {
+ synchronized (FsDatasetImpl.this) {
+ ramDiskReplicaTracker.recordEndLazyPersist(bpId, blockId, savedFiles);
+
+ targetVolume.incDfsUsed(bpId,
+ savedFiles[0].length() + savedFiles[1].length());
+
+ // Update metrics (ignore the metadata file size)
+ datanode.getMetrics().incrRamDiskBlocksLazyPersisted();
+ datanode.getMetrics().incrRamDiskBytesLazyPersisted(savedFiles[1].length());
+ datanode.getMetrics().addRamDiskBlocksLazyPersistWindowMs(
+ Time.monotonicNow() - creationTime);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter: Finish persisting RamDisk block: "
+ + " block pool Id: " + bpId + " block id: " + blockId
+ + " to block file " + savedFiles[1] + " and meta file " + savedFiles[0]
+ + " on target volume " + targetVolume);
+ }
+ }
+ }
+
+ @Override
+ public void onFailLazyPersist(String bpId, long blockId) {
+ RamDiskReplica block = null;
+ block = ramDiskReplicaTracker.getReplica(bpId, blockId);
+ if (block != null) {
+ LOG.warn("Failed to save replica " + block + ". re-enqueueing it.");
+ ramDiskReplicaTracker.reenqueueReplicaNotPersisted(block);
+ }
+ }
+
+ @Override
public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block,
FileDescriptor fd, long offset, long nbytes, int flags) {
FsVolumeImpl fsVolumeImpl = this.getVolume(block);
@@ -2316,9 +2361,38 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
nbytes, flags);
}
- void discardRamDiskReplica(RamDiskReplica replica, boolean deleteSavedCopies) {
- ramDiskReplicaTracker.discardReplica(replica.getBlockPoolId(),
- replica.getBlockId(), deleteSavedCopies);
+ private boolean ramDiskConfigured() {
+ for (FsVolumeImpl v: getVolumes()){
+ if (v.isTransientStorage()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Add/Remove per DISK volume async lazy persist thread when RamDisk volume is
+ // added or removed.
+ // This should only be called when the FsDataSetImpl#volumes list is finalized.
+ private void setupAsyncLazyPersistThreads() {
+ boolean ramDiskConfigured = ramDiskConfigured();
+ for (FsVolumeImpl v: getVolumes()){
+ // Skip transient volumes
+ if (v.isTransientStorage()) {
+ continue;
+ }
+
+ // Add thread for DISK volume if RamDisk is configured
+ if (ramDiskConfigured &&
+ !asyncLazyPersistService.queryVolume(v.getCurrentDir())) {
+ asyncLazyPersistService.addVolume(v.getCurrentDir());
+ }
+
+ // Remove thread for DISK volume if RamDisk is not configured
+ if (!ramDiskConfigured &&
+ asyncLazyPersistService.queryVolume(v.getCurrentDir())) {
+ asyncLazyPersistService.removeVolume(v.getCurrentDir());
+ }
+ }
}
class LazyWriter implements Runnable {
@@ -2344,61 +2418,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT);
}
- private void moveReplicaToNewVolume(String bpid, long blockId, long creationTime)
- throws IOException {
-
- FsVolumeImpl targetVolume;
- ReplicaInfo replicaInfo;
- BlockPoolSlice bpSlice;
- File srcFile, srcMeta;
- long genStamp;
-
- synchronized (FsDatasetImpl.this) {
- replicaInfo = volumeMap.get(bpid, blockId);
-
- if (replicaInfo == null || !replicaInfo.getVolume().isTransientStorage()) {
- // The block was either deleted before it could be checkpointed or
- // it is already on persistent storage. This can occur if a second
- // replica on persistent storage was found after the lazy write was
- // scheduled.
- return;
- }
-
- // Pick a target volume for the block.
- targetVolume = volumes.getNextVolume(
- StorageType.DEFAULT, replicaInfo.getNumBytes());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
- }
-
- ramDiskReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
- bpSlice = targetVolume.getBlockPoolSlice(bpid);
- srcMeta = replicaInfo.getMetaFile();
- srcFile = replicaInfo.getBlockFile();
- genStamp = replicaInfo.getGenerationStamp();
- }
-
- // Drop the FsDatasetImpl lock for the file copy.
- File[] savedFiles =
- bpSlice.lazyPersistReplica(blockId, genStamp, srcMeta, srcFile);
-
- synchronized (FsDatasetImpl.this) {
- ramDiskReplicaTracker.recordEndLazyPersist(bpid, blockId, savedFiles);
-
- // Update metrics (ignore the metadata file size)
- datanode.getMetrics().incrRamDiskBlocksLazyPersisted();
- datanode.getMetrics().incrRamDiskBytesLazyPersisted(replicaInfo.getNumBytes());
- datanode.getMetrics().addRamDiskBlocksLazyPersistWindowMs(
- Time.monotonicNow() - creationTime);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
- " to file " + savedFiles[1]);
- }
- }
- }
-
/**
* Checkpoint a pending replica to persistent storage now.
* If we fail then move the replica to the end of the queue.
@@ -2406,13 +2425,43 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
private boolean saveNextReplica() {
RamDiskReplica block = null;
+ FsVolumeImpl targetVolume;
+ ReplicaInfo replicaInfo;
boolean succeeded = false;
try {
block = ramDiskReplicaTracker.dequeueNextReplicaToPersist();
if (block != null) {
- moveReplicaToNewVolume(block.getBlockPoolId(), block.getBlockId(),
- block.getCreationTime());
+ synchronized (FsDatasetImpl.this) {
+ replicaInfo = volumeMap.get(block.getBlockPoolId(), block.getBlockId());
+
+ // If replicaInfo is null, the block was either deleted before
+ // it could be checkpointed or it is already on persistent storage.
+ // This can occur if a second replica on persistent storage was found
+ // after the lazy write was scheduled.
+ if (replicaInfo != null &&
+ replicaInfo.getVolume().isTransientStorage()) {
+ // Pick a target volume to persist the block.
+ targetVolume = volumes.getNextVolume(
+ StorageType.DEFAULT, replicaInfo.getNumBytes());
+
+ ramDiskReplicaTracker.recordStartLazyPersist(
+ block.getBlockPoolId(), block.getBlockId(), targetVolume);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter: Start persisting RamDisk block:"
+ + " block pool Id: " + block.getBlockPoolId()
+ + " block id: " + block.getBlockId()
+ + " on target volume " + targetVolume);
+ }
+
+ asyncLazyPersistService.submitLazyPersistTask(
+ block.getBlockPoolId(), block.getBlockId(),
+ replicaInfo.getGenerationStamp(), block.getCreationTime(),
+ replicaInfo.getMetaFile(), replicaInfo.getBlockFile(),
+ targetVolume);
+ }
+ }
}
succeeded = true;
} catch(IOException ioe) {
@@ -2420,10 +2469,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
} finally {
if (!succeeded && block != null) {
LOG.warn("Failed to save replica " + block + ". re-enqueueing it.");
- ramDiskReplicaTracker.reenqueueReplicaNotPersisted(block);
+ onFailLazyPersist(block.getBlockPoolId(), block.getBlockId());
}
}
-
return succeeded;
}
@@ -2480,7 +2528,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
metaFile = replicaInfo.getMetaFile();
blockFileUsed = blockFile.length();
metaFileUsed = metaFile.length();
- discardRamDiskReplica(replicaState, false);
+ ramDiskReplicaTracker.discardReplica(replicaState.getBlockPoolId(),
+ replicaState.getBlockId(), false);
// Move the replica from lazyPersist/ to finalized/ on target volume
BlockPoolSlice bpSlice =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 60ea125..32709be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -124,7 +124,11 @@ public class FsVolumeImpl implements FsVolumeSpi {
File getRbwDir(String bpid) throws IOException {
return getBlockPoolSlice(bpid).getRbwDir();
}
-
+
+ File getLazyPersistDir(String bpid) throws IOException {
+ return getBlockPoolSlice(bpid).getLazypersistDir();
+ }
+
void decDfsUsed(String bpid, long value) {
synchronized(dataset) {
BlockPoolSlice bp = bpSlices.get(bpid);
@@ -134,6 +138,15 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
}
+ void incDfsUsed(String bpid, long value) {
+ synchronized(dataset) {
+ BlockPoolSlice bp = bpSlices.get(bpid);
+ if (bp != null) {
+ bp.incDfsUsed(value);
+ }
+ }
+ }
+
long getDfsUsed() throws IOException {
long dfsUsed = 0;
synchronized(dataset) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
new file mode 100644
index 0000000..76acbea
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This class is a container of multiple thread pools, one for each non-RamDisk
+ * volume with a maximum thread count of 1 so that we can schedule async lazy
+ * persist operations easily with volume arrival and departure handled.
+ *
+ * This class and {@link org.apache.hadoop.util.AsyncDiskService} are similar.
+ * They should be combined.
+ */
+class RamDiskAsyncLazyPersistService {
+ public static final Log LOG = LogFactory.getLog(RamDiskAsyncLazyPersistService.class);
+
+ // ThreadPool core pool size
+ private static final int CORE_THREADS_PER_VOLUME = 1;
+ // ThreadPool maximum pool size
+ private static final int MAXIMUM_THREADS_PER_VOLUME = 1;
+ // ThreadPool keep-alive time for threads over core pool size
+ private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
+
+ private final DataNode datanode;
+ private final ThreadGroup threadGroup;
+ private Map<File, ThreadPoolExecutor> executors
+ = new HashMap<File, ThreadPoolExecutor>();
+
+ /**
+ * Create a RamDiskAsyncLazyPersistService with a set of volumes (specified by their
+ * root directories).
+ *
+ * The RamDiskAsyncLazyPersistService uses one ThreadPool per volume to do the async
+ * disk operations.
+ */
+ RamDiskAsyncLazyPersistService(DataNode datanode) {
+ this.datanode = datanode;
+ this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+ }
+
+ private void addExecutorForVolume(final File volume) {
+ ThreadFactory threadFactory = new ThreadFactory() {
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(threadGroup, r);
+ t.setName("Async RamDisk lazy persist worker for volume " + volume);
+ return t;
+ }
+ };
+
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(
+ CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+ THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), threadFactory);
+
+ // This can reduce the number of running threads
+ executor.allowCoreThreadTimeOut(true);
+ executors.put(volume, executor);
+ }
+
+ /**
+ * Starts AsyncLazyPersistService for a new volume
+ * @param volume the root of the new data volume.
+ */
+ synchronized void addVolume(File volume) {
+ if (executors == null) {
+ throw new RuntimeException("AsyncLazyPersistService is already shutdown");
+ }
+ ThreadPoolExecutor executor = executors.get(volume);
+ if (executor != null) {
+ throw new RuntimeException("Volume " + volume + " is already existed.");
+ }
+ addExecutorForVolume(volume);
+ }
+
+ /**
+ * Stops AsyncLazyPersistService for a volume.
+ * @param volume the root of the volume.
+ */
+ synchronized void removeVolume(File volume) {
+ if (executors == null) {
+ throw new RuntimeException("AsyncDiskService is already shutdown");
+ }
+ ThreadPoolExecutor executor = executors.get(volume);
+ if (executor == null) {
+ throw new RuntimeException("Can not find volume " + volume
+ + " to remove.");
+ } else {
+ executor.shutdown();
+ executors.remove(volume);
+ }
+ }
+
+ /**
+ * Query if the thread pool exist for the volume
+ * @param volume the root of a volume
+ * @return true if there is one thread pool for the volume
+ * false otherwise
+ */
+ synchronized boolean queryVolume(File volume) {
+ if (executors == null) {
+ throw new RuntimeException("AsyncLazyPersistService is already shutdown");
+ }
+ ThreadPoolExecutor executor = executors.get(volume);
+ return (executor != null);
+ }
+
+ /**
+ * Execute the task sometime in the future, using ThreadPools.
+ */
+ synchronized void execute(File root, Runnable task) {
+ if (executors == null) {
+ throw new RuntimeException("AsyncLazyPersistService is already shutdown");
+ }
+ ThreadPoolExecutor executor = executors.get(root);
+ if (executor == null) {
+ throw new RuntimeException("Cannot find root " + root
+ + " for execution of task " + task);
+ } else {
+ executor.execute(task);
+ }
+ }
+
+ /**
+ * Gracefully shut down all ThreadPool. Will wait for all lazy persist
+ * tasks to finish.
+ */
+ synchronized void shutdown() {
+ if (executors == null) {
+ LOG.warn("AsyncLazyPersistService has already shut down.");
+ } else {
+ LOG.info("Shutting down all async lazy persist service threads");
+
+ for (Map.Entry<File, ThreadPoolExecutor> e : executors.entrySet()) {
+ e.getValue().shutdown();
+ }
+ // clear the executor map so that calling execute again will fail.
+ executors = null;
+ LOG.info("All async lazy persist service threads have been shut down");
+ }
+ }
+
+ /**
+ * Asynchronously lazy persist the block from the RamDisk to Disk.
+ */
+ void submitLazyPersistTask(String bpId, long blockId,
+ long genStamp, long creationTime,
+ File metaFile, File blockFile,
+ FsVolumeImpl targetVolume) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter schedule async task to persist RamDisk block pool id: "
+ + bpId + " block id: " + blockId);
+ }
+
+ File lazyPersistDir = targetVolume.getLazyPersistDir(bpId);
+ if (!lazyPersistDir.exists() && !lazyPersistDir.mkdirs()) {
+ FsDatasetImpl.LOG.warn("LazyWriter failed to create " + lazyPersistDir);
+ throw new IOException("LazyWriter fail to find or create lazy persist dir: "
+ + lazyPersistDir.toString());
+ }
+
+ ReplicaLazyPersistTask lazyPersistTask = new ReplicaLazyPersistTask(
+ bpId, blockId, genStamp, creationTime, blockFile, metaFile,
+ targetVolume, lazyPersistDir);
+ execute(targetVolume.getCurrentDir(), lazyPersistTask);
+ }
+
+ class ReplicaLazyPersistTask implements Runnable {
+ final String bpId;
+ final long blockId;
+ final long genStamp;
+ final long creationTime;
+ final File blockFile;
+ final File metaFile;
+ final FsVolumeImpl targetVolume;
+ final File lazyPersistDir;
+
+ ReplicaLazyPersistTask(String bpId, long blockId,
+ long genStamp, long creationTime,
+ File blockFile, File metaFile,
+ FsVolumeImpl targetVolume, File lazyPersistDir) {
+ this.bpId = bpId;
+ this.blockId = blockId;
+ this.genStamp = genStamp;
+ this.creationTime = creationTime;
+ this.blockFile = blockFile;
+ this.metaFile = metaFile;
+ this.targetVolume = targetVolume;
+ this.lazyPersistDir = lazyPersistDir;
+ }
+
+ @Override
+ public String toString() {
+ // Called in AsyncLazyPersistService.execute for displaying error messages.
+ return "LazyWriter async task of persist RamDisk block pool id:"
+ + bpId + " block pool id: "
+ + blockId + " with block file " + blockFile
+ + " and meta file " + metaFile + " to target volume " + targetVolume;}
+
+ @Override
+ public void run() {
+ boolean succeeded = false;
+ try {
+ // No FsDatasetImpl lock for the file copy
+ File targetFiles[] = FsDatasetImpl.copyBlockFiles(
+ blockId, genStamp, metaFile, blockFile, lazyPersistDir);
+
+ // Lock FsDataSetImpl during onCompleteLazyPersist callback
+ datanode.getFSDataset().onCompleteLazyPersist(bpId, blockId,
+ creationTime, targetFiles, targetVolume);
+ succeeded = true;
+ } catch (Exception e){
+ FsDatasetImpl.LOG.warn(
+ "LazyWriter failed to async persist RamDisk block pool id: "
+ + bpId + "block Id: " + blockId);
+ } finally {
+ if (!succeeded) {
+ datanode.getFSDataset().onFailLazyPersist(bpId, blockId);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index d1284fe..0786bc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -1209,5 +1210,16 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
FileDescriptor fd, long offset, long nbytes, int flags) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public void onCompleteLazyPersist(String bpId, long blockId,
+ long creationTime, File[] savedFiles, FsVolumeImpl targetVolume) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void onFailLazyPersist(String bpId, long blockId) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b6d115c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 91deb55..9f1d50a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -971,7 +971,9 @@ public class TestLazyPersistFiles {
void printRamDiskJMXMetrics() {
try {
- jmx.printAllMatchedAttributes(JMX_RAM_DISK_METRICS_PATTERN);
+ if (jmx != null) {
+ jmx.printAllMatchedAttributes(JMX_RAM_DISK_METRICS_PATTERN);
+ }
} catch (Exception e) {
e.printStackTrace();
}
[32/34] git commit: HDFS-6921. Add LazyPersist flag to FileStatus.
(Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6921. Add LazyPersist flag to FileStatus. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9183dadb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9183dadb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9183dadb
Branch: refs/heads/branch-2.6
Commit: 9183dadb738bac5bf5ad166108dd7875a2940ed9
Parents: 2e98379
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 08:52:55 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:53 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/CreateFlag.java | 16 +++++++++++++++-
.../org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 7 +++++++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 1 -
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 5 +++--
.../hadoop/hdfs/server/namenode/INodeFile.java | 6 ------
.../hdfs/server/namenode/INodeFileAttributes.java | 3 ---
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
.../hdfs/server/datanode/TestDirectoryScanner.java | 1 +
.../datanode/fsdataset/impl/TestFsDatasetImpl.java | 2 +-
9 files changed, 28 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index 252f37b..c5d23b4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -79,7 +79,21 @@ public enum CreateFlag {
/**
* Force closed blocks to disk. Similar to POSIX O_SYNC. See javadoc for description.
*/
- SYNC_BLOCK((short) 0x08);
+ SYNC_BLOCK((short) 0x08),
+
+ /**
+ * Create the block on transient storage (RAM) if available. If
+ * transient storage is unavailable then the block will be created
+ * on disk.
+ *
+ * HDFS will make a best effort to lazily write these files to persistent
+ * storage, however file contents may be lost at any time due to process/
+ * node restarts, hence there is no guarantee of data durability.
+ *
+ * This flag must only be used for intermediate data whose loss can be
+ * tolerated by the application.
+ */
+ LAZY_PERSIST((short) 0x10);
private final short mode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 8e5e8cf..83c2a62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -1363,6 +1363,9 @@ public class PBHelper {
if (flag.contains(CreateFlag.OVERWRITE)) {
value |= CreateFlagProto.OVERWRITE.getNumber();
}
+ if (flag.contains(CreateFlag.LAZY_PERSIST)) {
+ value |= CreateFlagProto.LAZY_PERSIST.getNumber();
+ }
return value;
}
@@ -1379,6 +1382,10 @@ public class PBHelper {
== CreateFlagProto.OVERWRITE_VALUE) {
result.add(CreateFlag.OVERWRITE);
}
+ if ((flag & CreateFlagProto.LAZY_PERSIST_VALUE)
+ == CreateFlagProto.LAZY_PERSIST_VALUE) {
+ result.add(CreateFlag.LAZY_PERSIST);
+ }
return new EnumSetWritable<CreateFlag>(result);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 3e6cf5d..9ca50c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2431,7 +2431,6 @@ public class FSDirectory implements Closeable {
blocksize = fileNode.getPreferredBlockSize();
isEncrypted = (feInfo != null) ||
(isRawPath && isInAnEZ(INodesInPath.fromINode(node)));
- isLazyPersist = fileNode.getLazyPersistFlag();
} else {
isEncrypted = isInAnEZ(INodesInPath.fromINode(node));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 639faf3..216e0c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2565,6 +2565,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+ boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST);
waitForLoadingFSImage();
@@ -2624,7 +2625,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
src = resolvePath(src, pathComponents);
toRemoveBlocks = startFileInternal(pc, src, permissions, holder,
clientMachine, create, overwrite, createParent, replication,
- blockSize, suite, protocolVersion, edek, logRetryCache);
+ blockSize, isLazyPersist, suite, protocolVersion, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
FSDirectory.isReservedRawName(srcArg), true);
} catch (StandbyException se) {
@@ -2660,7 +2661,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
String src, PermissionStatus permissions, String holder,
String clientMachine, boolean create, boolean overwrite,
boolean createParent, short replication, long blockSize,
- CipherSuite suite, CryptoProtocolVersion version,
+ boolean isLazyPersist, CipherSuite suite, CryptoProtocolVersion version,
EncryptedKeyVersion edek, boolean logRetryEntry)
throws FileAlreadyExistsException, AccessControlException,
UnresolvedLinkException, FileNotFoundException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index b04c0bb..5136f8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -108,9 +108,6 @@ public class INodeFile extends INodeWithAdditionalFields
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
return h;
}
- static boolean getLazyPersistFlag(long header) {
- return LAZY_PERSIST.BITS.retrieve(header) == 0 ? false : true;
- }
}
private long header = 0L;
@@ -402,9 +399,6 @@ public class INodeFile extends INodeWithAdditionalFields
recordModification(latestSnapshotId);
setStoragePolicyID(storagePolicyId);
}
- public boolean getLazyPersistFlag() {
- return HeaderFormat.getLazyPersistFlag(header);
- }
@Override
public long getHeaderLong() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
index 9577d1e..b8af50d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
@@ -74,9 +74,6 @@ public interface INodeFileAttributes extends INodeAttributes {
}
@Override
- public boolean getLazyPersistFlag() { return HeaderFormat.getLazyPersistFlag(header); }
-
- @Override
public long getHeaderLong() {
return header;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
index 14e8c68..232c264 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
@@ -65,6 +65,7 @@ enum CreateFlagProto {
CREATE = 0x01; // Create a file
OVERWRITE = 0x02; // Truncate/overwrite a file. Same as POSIX O_TRUNC
APPEND = 0x04; // Append to a file
+ LAZY_PERSIST = 0x10; // File with reduced durability guarantees.
}
message CreateRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index f27e117..be034fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -576,6 +576,7 @@ public class TestDirectoryScanner {
@Override
public void releaseReservedSpace(long bytesToRelease) {
+ }
@Override
public boolean isTransientStorage() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9183dadb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 10b9f7e..7c39ca5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -147,7 +147,7 @@ public class TestFsDatasetImpl {
for (int i = 0; i < NUM_BLOCKS; i++) {
String bpid = BLOCK_POOL_IDS[NUM_BLOCKS % BLOCK_POOL_IDS.length];
ExtendedBlock eb = new ExtendedBlock(bpid, i);
- dataset.createRbw(StorageType.DEFAULT, eb);
+ dataset.createRbw(StorageType.DEFAULT, eb, false);
}
final String[] dataDirs =
conf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(",");
[30/34] git commit: HDFS-7129. Metrics to track usage of memory for
writes. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-7129. Metrics to track usage of memory for writes. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6457a0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6457a0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6457a0e
Branch: refs/heads/branch-2.6
Commit: e6457a0e0816f9a865b190722767dc01120c62f7
Parents: d61daff
Author: arp <ar...@apache.org>
Authored: Tue Sep 30 00:53:18 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:53 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 40 +++++++++-
.../impl/RamDiskReplicaLruTracker.java | 20 ++++-
.../fsdataset/impl/RamDiskReplicaTracker.java | 23 +++++-
.../datanode/metrics/DataNodeMetrics.java | 80 ++++++++++++++++++++
.../org/apache/hadoop/hdfs/tools/JMXGet.java | 18 +++++
.../fsdataset/impl/TestLazyPersistFiles.java | 80 ++++++++++++++++----
6 files changed, 238 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 8ed81df..f2daf99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1013,11 +1013,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (allowLazyPersist) {
// First try to place the block on a transient volume.
v = volumes.getNextTransientVolume(b.getNumBytes());
+ datanode.getMetrics().incrRamDiskBlocksWrite();
} else {
v = volumes.getNextVolume(storageType, b.getNumBytes());
}
} catch (DiskOutOfSpaceException de) {
if (allowLazyPersist) {
+ datanode.getMetrics().incrRamDiskBlocksWriteFallback();
allowLazyPersist = false;
continue;
}
@@ -1245,6 +1247,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (v.isTransientStorage()) {
ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
+ datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes());
}
}
volumeMap.add(bpid, newReplicaInfo);
@@ -1500,7 +1503,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
if (v.isTransientStorage()) {
- ramDiskReplicaTracker.discardReplica(bpid, invalidBlks[i].getBlockId(), true);
+ RamDiskReplica replicaInfo =
+ ramDiskReplicaTracker.getReplica(bpid, invalidBlks[i].getBlockId());
+ if (replicaInfo != null) {
+ if (replicaInfo.getIsPersisted() == false) {
+ datanode.getMetrics().incrRamDiskBlocksDeletedBeforeLazyPersisted();
+ }
+ discardRamDiskReplica(replicaInfo, true);
+ }
}
// If a DFSClient has the replica in its cache of short-circuit file
@@ -1646,11 +1656,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (info != null) {
if (touch && info.getVolume().isTransientStorage()) {
ramDiskReplicaTracker.touch(bpid, blockId);
+ datanode.getMetrics().incrRamDiskBlocksReadHits();
}
return info.getBlockFile();
}
return null;
}
+
/**
* check if a data directory is healthy
* if some volumes failed - make sure to remove all the blocks that belong
@@ -2304,6 +2316,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
nbytes, flags);
}
+ void discardRamDiskReplica(RamDiskReplica replica, boolean deleteSavedCopies) {
+ ramDiskReplicaTracker.discardReplica(replica.getBlockPoolId(),
+ replica.getBlockId(), deleteSavedCopies);
+ }
+
class LazyWriter implements Runnable {
private volatile boolean shouldRun = true;
final int checkpointerInterval;
@@ -2327,7 +2344,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT);
}
- private void moveReplicaToNewVolume(String bpid, long blockId)
+ private void moveReplicaToNewVolume(String bpid, long blockId, long creationTime)
throws IOException {
FsVolumeImpl targetVolume;
@@ -2369,6 +2386,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
synchronized (FsDatasetImpl.this) {
ramDiskReplicaTracker.recordEndLazyPersist(bpid, blockId, savedFiles);
+ // Update metrics (ignore the metadata file size)
+ datanode.getMetrics().incrRamDiskBlocksLazyPersisted();
+ datanode.getMetrics().incrRamDiskBytesLazyPersisted(replicaInfo.getNumBytes());
+ datanode.getMetrics().addRamDiskBlocksLazyPersistWindowMs(
+ Time.monotonicNow() - creationTime);
+
if (LOG.isDebugEnabled()) {
LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
" to file " + savedFiles[1]);
@@ -2388,7 +2411,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
try {
block = ramDiskReplicaTracker.dequeueNextReplicaToPersist();
if (block != null) {
- moveReplicaToNewVolume(block.getBlockPoolId(), block.getBlockId());
+ moveReplicaToNewVolume(block.getBlockPoolId(), block.getBlockId(),
+ block.getCreationTime());
}
succeeded = true;
} catch(IOException ioe) {
@@ -2456,7 +2480,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
metaFile = replicaInfo.getMetaFile();
blockFileUsed = blockFile.length();
metaFileUsed = metaFile.length();
- ramDiskReplicaTracker.discardReplica(replicaState, false);
+ discardRamDiskReplica(replicaState, false);
// Move the replica from lazyPersist/ to finalized/ on target volume
BlockPoolSlice bpSlice =
@@ -2474,6 +2498,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Update the volumeMap entry.
volumeMap.add(bpid, newReplicaInfo);
+
+ // Update metrics
+ datanode.getMetrics().incrRamDiskBlocksEvicted();
+ datanode.getMetrics().addRamDiskBlocksEvictionWindowMs(
+ Time.monotonicNow() - replicaState.getCreationTime());
+ if (replicaState.getNumReads() == 0) {
+ datanode.getMetrics().incrRamDiskBlocksEvictedWithoutRead();
+ }
}
// Before deleting the files from transient storage we must notify the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
index 7808003..a843d9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import com.google.common.collect.TreeMultimap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.Time;
import java.io.File;
import java.util.*;
@@ -97,9 +98,11 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
return;
}
+ ramDiskReplicaLru.numReads.getAndIncrement();
+
// Reinsert the replica with its new timestamp.
if (replicasPersisted.remove(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru)) {
- ramDiskReplicaLru.lastUsedTime = System.currentTimeMillis();
+ ramDiskReplicaLru.lastUsedTime = Time.monotonicNow();
replicasPersisted.put(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru);
}
}
@@ -132,8 +135,9 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
replicasNotPersisted.remove(ramDiskReplicaLru);
}
- ramDiskReplicaLru.lastUsedTime = System.currentTimeMillis();
+ ramDiskReplicaLru.lastUsedTime = Time.monotonicNow();
replicasPersisted.put(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru);
+ ramDiskReplicaLru.isPersisted = true;
}
@Override
@@ -215,4 +219,16 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
// replicasNotPersisted will be lazily GC'ed.
}
+
+ @Override
+ synchronized RamDiskReplica getReplica(
+ final String bpid, final long blockId) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+
+ if (map == null) {
+ return null;
+ }
+
+ return map.get(blockId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
index 2401424..7507925 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
@@ -28,8 +28,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
import java.io.File;
+import java.util.concurrent.atomic.AtomicLong;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -44,6 +46,10 @@ public abstract class RamDiskReplicaTracker {
private File savedBlockFile;
private File savedMetaFile;
+ private long creationTime;
+ protected AtomicLong numReads = new AtomicLong(0);
+ protected boolean isPersisted;
+
/**
* RAM_DISK volume that holds the original replica.
*/
@@ -62,6 +68,8 @@ public abstract class RamDiskReplicaTracker {
lazyPersistVolume = null;
savedMetaFile = null;
savedBlockFile = null;
+ creationTime = Time.monotonicNow();
+ isPersisted = false;
}
long getBlockId() {
@@ -89,6 +97,12 @@ public abstract class RamDiskReplicaTracker {
return savedMetaFile;
}
+ long getNumReads() { return numReads.get(); }
+
+ long getCreationTime() { return creationTime; }
+
+ boolean getIsPersisted() {return isPersisted; }
+
/**
* Record the saved meta and block files on the given volume.
*
@@ -243,7 +257,10 @@ public abstract class RamDiskReplicaTracker {
final String bpid, final long blockId,
boolean deleteSavedCopies);
- void discardReplica(RamDiskReplica replica, boolean deleteSavedCopies) {
- discardReplica(replica.getBlockPoolId(), replica.getBlockId(), deleteSavedCopies);
- }
+ /**
+ * Return RamDiskReplica info given block pool id and block id
+ * Return null if it does not exist in RamDisk
+ */
+ abstract RamDiskReplica getReplica(
+ final String bpid, final long blockId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index b536e7e..57f12db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -65,6 +65,26 @@ public class DataNodeMetrics {
@Metric MutableCounterLong writesFromRemoteClient;
@Metric MutableCounterLong blocksGetLocalPathInfo;
+ // RamDisk metrics on read/write
+ @Metric MutableCounterLong ramDiskBlocksWrite;
+ @Metric MutableCounterLong ramDiskBlocksWriteFallback;
+ @Metric MutableCounterLong ramDiskBytesWrite;
+ @Metric MutableCounterLong ramDiskBlocksReadHits;
+
+ // RamDisk metrics on eviction
+ @Metric MutableCounterLong ramDiskBlocksEvicted;
+ @Metric MutableCounterLong ramDiskBlocksEvictedWithoutRead;
+ @Metric MutableRate ramDiskBlocksEvictionWindowMs;
+ final MutableQuantiles[] ramDiskBlocksEvictionWindowMsQuantiles;
+
+
+ // RamDisk metrics on lazy persist
+ @Metric MutableCounterLong ramDiskBlocksLazyPersisted;
+ @Metric MutableCounterLong ramDiskBlocksDeletedBeforeLazyPersisted;
+ @Metric MutableCounterLong ramDiskBytesLazyPersisted;
+ @Metric MutableRate ramDiskBlocksLazyPersistWindowMs;
+ final MutableQuantiles[] ramDiskBlocksLazyPersistWindowMsQuantiles;
+
@Metric MutableCounterLong fsyncCount;
@Metric MutableCounterLong volumeFailures;
@@ -107,6 +127,8 @@ public class DataNodeMetrics {
fsyncNanosQuantiles = new MutableQuantiles[len];
sendDataPacketBlockedOnNetworkNanosQuantiles = new MutableQuantiles[len];
sendDataPacketTransferNanosQuantiles = new MutableQuantiles[len];
+ ramDiskBlocksEvictionWindowMsQuantiles = new MutableQuantiles[len];
+ ramDiskBlocksLazyPersistWindowMsQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
@@ -127,6 +149,14 @@ public class DataNodeMetrics {
"sendDataPacketTransferNanos" + interval + "s",
"Time reading from disk and writing to network while sending " +
"a packet in ns", "ops", "latency", interval);
+ ramDiskBlocksEvictionWindowMsQuantiles[i] = registry.newQuantiles(
+ "ramDiskBlocksEvictionWindows" + interval + "s",
+ "Time between the RamDisk block write and eviction in ms",
+ "ops", "latency", interval);
+ ramDiskBlocksLazyPersistWindowMsQuantiles[i] = registry.newQuantiles(
+ "ramDiskBlocksLazyPersistWindows" + interval + "s",
+ "Time between the RamDisk block write and disk persist in ms",
+ "ops", "latency", interval);
}
}
@@ -284,4 +314,54 @@ public class DataNodeMetrics {
q.add(latencyNanos);
}
}
+
+ public void incrRamDiskBlocksWrite() {
+ ramDiskBlocksWrite.incr();
+ }
+
+ public void incrRamDiskBlocksWriteFallback() {
+ ramDiskBlocksWriteFallback.incr();
+ }
+
+ public void addRamDiskBytesWrite(long bytes) {
+ ramDiskBytesWrite.incr(bytes);
+ }
+
+ public void incrRamDiskBlocksReadHits() {
+ ramDiskBlocksReadHits.incr();
+ }
+
+ public void incrRamDiskBlocksEvicted() {
+ ramDiskBlocksEvicted.incr();
+ }
+
+ public void incrRamDiskBlocksEvictedWithoutRead() {
+ ramDiskBlocksEvictedWithoutRead.incr();
+ }
+
+ public void addRamDiskBlocksEvictionWindowMs(long latencyMs) {
+ ramDiskBlocksEvictionWindowMs.add(latencyMs);
+ for (MutableQuantiles q : ramDiskBlocksEvictionWindowMsQuantiles) {
+ q.add(latencyMs);
+ }
+ }
+
+ public void incrRamDiskBlocksLazyPersisted() {
+ ramDiskBlocksLazyPersisted.incr();
+ }
+
+ public void incrRamDiskBlocksDeletedBeforeLazyPersisted() {
+ ramDiskBlocksDeletedBeforeLazyPersisted.incr();
+ }
+
+ public void incrRamDiskBytesLazyPersisted(long bytes) {
+ ramDiskBytesLazyPersisted.incr(bytes);
+ }
+
+ public void addRamDiskBlocksLazyPersistWindowMs(long latencyMs) {
+ ramDiskBlocksLazyPersistWindowMs.add(latencyMs);
+ for (MutableQuantiles q : ramDiskBlocksLazyPersistWindowMsQuantiles) {
+ q.add(latencyMs);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
index bafef25..bbd545a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/JMXGet.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Set;
import java.util.TreeSet;
+import java.util.regex.Pattern;
import javax.management.AttributeNotFoundException;
import javax.management.MBeanAttributeInfo;
@@ -109,6 +110,23 @@ public class JMXGet {
}
}
+ public void printAllMatchedAttributes(String attrRegExp) throws Exception {
+ err("List of the keys matching " + attrRegExp + " :");
+ Object val = null;
+ Pattern p = Pattern.compile(attrRegExp);
+ for (ObjectName oname : hadoopObjectNames) {
+ err(">>>>>>>>jmx name: " + oname.getCanonicalKeyPropertyListString());
+ MBeanInfo mbinfo = mbsc.getMBeanInfo(oname);
+ MBeanAttributeInfo[] mbinfos = mbinfo.getAttributes();
+ for (MBeanAttributeInfo mb : mbinfos) {
+ if (p.matcher(mb.getName()).lookingAt()) {
+ val = mbsc.getAttribute(oname, mb.getName());
+ System.out.format(format, mb.getName(), (val == null) ? "" : val.toString());
+ }
+ }
+ }
+ }
+
/**
* get single value by key
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6457a0e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 928d0d0..91deb55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -31,19 +31,18 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.tools.JMXGet;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -58,6 +57,7 @@ import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class TestLazyPersistFiles {
@@ -81,14 +81,21 @@ public class TestLazyPersistFiles {
private static final int LAZY_WRITER_INTERVAL_SEC = 1;
private static final int BUFFER_LENGTH = 4096;
private static final int EVICTION_LOW_WATERMARK = 1;
+ private static final String JMX_SERVICE_NAME = "DataNode";
+ private static final String JMX_RAM_DISK_METRICS_PATTERN = "^RamDisk";
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private DFSClient client;
private Configuration conf;
+ private JMXGet jmx;
@After
- public void shutDownCluster() throws IOException {
+ public void shutDownCluster() throws Exception {
+
+ // Dump all RamDisk JMX metrics before shutdown the cluster
+ printRamDiskJMXMetrics();
+
if (fs != null) {
fs.close();
fs = null;
@@ -100,6 +107,10 @@ public class TestLazyPersistFiles {
cluster.shutdown();
cluster = null;
}
+
+ if (jmx != null) {
+ jmx = null;
+ }
}
@Test (timeout=300000)
@@ -203,13 +214,15 @@ public class TestLazyPersistFiles {
* @throws IOException
*/
@Test (timeout=300000)
- public void testFallbackToDiskFull() throws IOException {
+ public void testFallbackToDiskFull() throws Exception {
startUpCluster(false, 0);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
makeTestFile(path, BLOCK_SIZE, true);
ensureFileReplicasOnStorageType(path, DEFAULT);
+
+ verifyRamDiskJMXMetric("RamDiskBlocksWriteFallback", 1);
}
/**
@@ -384,11 +397,10 @@ public class TestLazyPersistFiles {
/**
* RamDisk eviction after lazy persist to disk.
- * @throws IOException
- * @throws InterruptedException
+ * @throws Exception
*/
@Test (timeout=300000)
- public void testRamDiskEviction() throws IOException, InterruptedException {
+ public void testRamDiskEviction() throws Exception {
startUpCluster(true, 1 + EVICTION_LOW_WATERMARK);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -411,6 +423,9 @@ public class TestLazyPersistFiles {
// RAM_DISK.
ensureFileReplicasOnStorageType(path2, RAM_DISK);
ensureFileReplicasOnStorageType(path1, DEFAULT);
+
+ verifyRamDiskJMXMetric("RamDiskBlocksEvicted", 1);
+ verifyRamDiskJMXMetric("RamDiskBlocksEvictedWithoutRead", 1);
}
/**
@@ -454,7 +469,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testRamDiskEvictionIsLru()
- throws IOException, InterruptedException {
+ throws Exception {
final int NUM_PATHS = 5;
startUpCluster(true, NUM_PATHS + EVICTION_LOW_WATERMARK);
final String METHOD_NAME = GenericTestUtils.getMethodName();
@@ -499,6 +514,14 @@ public class TestLazyPersistFiles {
ensureFileReplicasOnStorageType(paths[indexes.get(j)], RAM_DISK);
}
}
+
+ verifyRamDiskJMXMetric("RamDiskBlocksWrite", NUM_PATHS * 2);
+ verifyRamDiskJMXMetric("RamDiskBlocksWriteFallback", 0);
+ verifyRamDiskJMXMetric("RamDiskBytesWrite", BLOCK_SIZE * NUM_PATHS * 2);
+ verifyRamDiskJMXMetric("RamDiskBlocksReadHits", NUM_PATHS);
+ verifyRamDiskJMXMetric("RamDiskBlocksEvicted", NUM_PATHS);
+ verifyRamDiskJMXMetric("RamDiskBlocksEvictedWithoutRead", 0);
+ verifyRamDiskJMXMetric("RamDiskBlocksDeletedBeforeLazyPersisted", 0);
}
/**
@@ -506,9 +529,9 @@ public class TestLazyPersistFiles {
* Memory is freed up and file is gone.
* @throws IOException
*/
- @Test (timeout=300000)
+ @Test // (timeout=300000)
public void testDeleteBeforePersist()
- throws IOException, InterruptedException {
+ throws Exception {
startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));
@@ -523,6 +546,8 @@ public class TestLazyPersistFiles {
Assert.assertFalse(fs.exists(path));
assertThat(verifyDeletedBlocks(locatedBlocks), is(true));
+
+ verifyRamDiskJMXMetric("RamDiskBlocksDeletedBeforeLazyPersisted", 1);
}
/**
@@ -533,7 +558,7 @@ public class TestLazyPersistFiles {
*/
@Test (timeout=300000)
public void testDeleteAfterPersist()
- throws IOException, InterruptedException {
+ throws Exception {
startUpCluster(true, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -548,9 +573,10 @@ public class TestLazyPersistFiles {
client.delete(path.toString(), false);
Assert.assertFalse(fs.exists(path));
- triggerBlockReport();
-
assertThat(verifyDeletedBlocks(locatedBlocks), is(true));
+
+ verifyRamDiskJMXMetric("RamDiskBlocksLazyPersisted", 1);
+ verifyRamDiskJMXMetric("RamDiskBytesLazyPersisted", BLOCK_SIZE);
}
/**
@@ -760,6 +786,11 @@ public class TestLazyPersistFiles {
.build();
fs = cluster.getFileSystem();
client = fs.getClient();
+ try {
+ jmx = initJMX();
+ } catch (Exception e) {
+ fail("Failed initialize JMX for testing: " + e);
+ }
LOG.info("Cluster startup complete");
}
@@ -929,4 +960,25 @@ public class TestLazyPersistFiles {
}
}
}
+
+ JMXGet initJMX() throws Exception
+ {
+ JMXGet jmx = new JMXGet();
+ jmx.setService(JMX_SERVICE_NAME);
+ jmx.init();
+ return jmx;
+ }
+
+ void printRamDiskJMXMetrics() {
+ try {
+ jmx.printAllMatchedAttributes(JMX_RAM_DISK_METRICS_PATTERN);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ void verifyRamDiskJMXMetric(String metricName, long expectedValue)
+ throws Exception {
+ assertEquals(expectedValue, Integer.parseInt(jmx.getValue(metricName)));
+ }
}
[14/34] git commit: HDFS-6930. Improve replica eviction from RAM
disk. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6930. Improve replica eviction from RAM disk. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34f28eaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34f28eaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34f28eaf
Branch: refs/heads/branch-2.6
Commit: 34f28eaff9c4386dbfffdb9fc7640c32dae01eb6
Parents: 4609ba0
Author: arp <ar...@apache.org>
Authored: Wed Sep 3 13:53:01 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 214 +++++++++----------
.../fsdataset/impl/LazyWriteReplicaTracker.java | 52 +++--
2 files changed, 137 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34f28eaf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 797dcc6..8d93dcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -996,83 +996,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
- /**
- * Attempt to evict one or more transient block replicas we have at least
- * spaceNeeded bytes free.
- *
- * @return true if we were able to free up at least spaceNeeded bytes, false
- * otherwise.
- */
- private boolean tryToEvictBlocks(final String bpid, final long spaceNeeded)
- throws IOException {
-
- boolean isAvailable = false;
-
- LOG.info("Attempting to evict blocks from transient storage");
-
- // Reverse the map so we can iterate in order of replica creation times,
- // evicting oldest replicas one at a time until we have sufficient space.
- TreeMultimap<Long, LazyWriteReplicaTracker.ReplicaState> lruMap =
- lazyWriteReplicaTracker.getLruMap();
- int blocksEvicted = 0;
-
- // TODO: It is really inefficient to do this with the Object lock held!
- // TODO: This logic is here just for prototyping.
- // TODO: We should replace it with proactive discard when ram_disk free space
- // TODO: falls below a low watermark. That way we avoid fs operations on the
- // TODO: hot path with the lock held.
- synchronized (this) {
- long currentTime = System.currentTimeMillis() / 1000;
- for (Map.Entry<Long, LazyWriteReplicaTracker.ReplicaState> entry : lruMap.entries()) {
- LazyWriteReplicaTracker.ReplicaState lazyWriteReplica = entry.getValue();
- LOG.info("RAM_DISK: Evicting blockId=" + lazyWriteReplica.blockId +
- "; block LMT=" + entry.getKey() +
- "; currentTime=" + currentTime);
- ReplicaInfo replicaInfo = getReplicaInfo(bpid, lazyWriteReplica.blockId);
- Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
- File blockFile = replicaInfo.getBlockFile();
- File metaFile = replicaInfo.getMetaFile();
- long used = blockFile.length() + metaFile.length();
- lazyWriteReplicaTracker.discardReplica(bpid, entry.getValue().blockId, false);
-
- // Move the persisted replica to the finalized directory of
- // the target volume.
- BlockPoolSlice bpSlice =
- lazyWriteReplica.lazyPersistVolume.getBlockPoolSlice(bpid);
- File newBlockFile = bpSlice.activateSavedReplica(
- replicaInfo, lazyWriteReplica.savedBlockFile);
-
- ReplicaInfo newReplicaInfo =
- new FinalizedReplica(replicaInfo.getBlockId(),
- replicaInfo.getBytesOnDisk(),
- replicaInfo.getGenerationStamp(),
- lazyWriteReplica.lazyPersistVolume,
- newBlockFile.getParentFile());
-
- // Update the volumeMap entry. This removes the old entry.
- volumeMap.add(bpid, newReplicaInfo);
-
- // Remove the old replicas.
- blockFile.delete();
- metaFile.delete();
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, used);
- ++blocksEvicted;
-
- if (replicaInfo.getVolume().getAvailable() > spaceNeeded) {
- LOG.info("RAM_DISK: freed up " + spaceNeeded + " bytes for new block");
- isAvailable = true;
- break;
- }
-
- if (blocksEvicted == MAX_BLOCK_EVICTIONS_PER_ITERATION) {
- break;
- }
- }
- }
-
- return isAvailable;
- }
-
@Override // FsDatasetSpi
public synchronized ReplicaInPipeline createRbw(StorageType storageType,
ExtendedBlock b, boolean allowLazyPersist) throws IOException {
@@ -1095,13 +1018,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
} catch (DiskOutOfSpaceException de) {
if (allowLazyPersist) {
- if (!tryToEvictBlocks(b.getBlockPoolId(), b.getNumBytes())) {
- // Eviction did not work, we'll just fallback to DEFAULT storage.
- LOG.info("RAM_DISK: Failed to free up " + b.getNumBytes() +
- " bytes for new block. Will fallback to DEFAULT " +
- "storage");
- allowLazyPersist = false;
- }
+ allowLazyPersist = false;
continue;
}
throw de;
@@ -2376,20 +2293,26 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
class LazyWriter implements Runnable {
private volatile boolean shouldRun = true;
final int checkpointerInterval;
+ final long estimateBlockSize;
+
+ public static final int LOW_WATERMARK_FREE_SPACE_PERCENT = 10;
+ public static final int LOW_WATERMARK_FREE_SPACE_REPLICAS = 3;
+
public LazyWriter(final int checkpointerInterval) {
this.checkpointerInterval = checkpointerInterval;
+ this.estimateBlockSize = conf.getLongBytes(
+ DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
}
private void moveReplicaToNewVolume(String bpid, long blockId)
throws IOException {
- LOG.info("LazyWriter invoked to save blockId=" + blockId + "; bpid=" + bpid);
-
FsVolumeImpl targetVolume;
ReplicaInfo replicaInfo;
- synchronized (this) {
+ synchronized (FsDatasetImpl.this) {
replicaInfo = volumeMap.get(bpid, blockId);
if (replicaInfo == null || !replicaInfo.getVolume().isTransientStorage()) {
@@ -2403,20 +2326,20 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Pick a target volume for the block.
targetVolume = volumes.getNextVolume(
StorageType.DEFAULT, replicaInfo.getNumBytes());
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
+ }
- lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
- File savedBlockFile = targetVolume.getBlockPoolSlice(bpid)
- .lazyPersistReplica(replicaInfo);
- lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedBlockFile);
+ lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
+ File savedBlockFile = targetVolume.getBlockPoolSlice(bpid)
+ .lazyPersistReplica(replicaInfo);
+ lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedBlockFile);
- if (LOG.isDebugEnabled()) {
- LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
- " to file " + savedBlockFile);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
+ " to file " + savedBlockFile);
+ }
}
}
@@ -2430,28 +2353,100 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
boolean succeeded = false;
try {
- synchronized (this) {
- replicaState = lazyWriteReplicaTracker.dequeueNextReplicaToPersist();
- if (replicaState == null) {
- return false;
- }
+ replicaState = lazyWriteReplicaTracker.dequeueNextReplicaToPersist();
+ if (replicaState != null) {
+ // Move the replica outside the lock.
+ moveReplicaToNewVolume(replicaState.bpid, replicaState.blockId);
}
-
- // Move the replica outside the lock.
- moveReplicaToNewVolume(replicaState.bpid, replicaState.blockId);
succeeded = true;
} catch(IOException ioe) {
LOG.warn("Exception saving replica " + replicaState, ioe);
} finally {
if (!succeeded && replicaState != null) {
LOG.warn("Failed to save replica " + replicaState + ". re-enqueueing it.");
- lazyWriteReplicaTracker.reenqueueReplica(replicaState);
+ lazyWriteReplicaTracker.reenqueueReplicaNotPersisted(replicaState);
}
}
return succeeded;
}
+ private boolean transientFreeSpaceBelowThreshold() throws IOException {
+ long free = 0;
+ long capacity = 0;
+
+ // Don't worry about fragmentation for now. We don't expect more than one
+ // transient volume per DN.
+ for (FsVolumeImpl v : volumes.volumes) {
+ if (v.isTransientStorage()) {
+ capacity += v.getCapacity();
+ free += v.getAvailable();
+ }
+ }
+
+ if (capacity == 0) {
+ return false;
+ }
+
+ int percentFree = (int) (free * 100 / capacity);
+ return percentFree < LOW_WATERMARK_FREE_SPACE_PERCENT ||
+ free < (estimateBlockSize * LOW_WATERMARK_FREE_SPACE_REPLICAS);
+ }
+
+ /**
+ * Attempt to evict one or more transient block replicas we have at least
+ * spaceNeeded bytes free.
+ */
+ private synchronized void evictBlocks() throws IOException {
+ int iterations = 0;
+
+ LazyWriteReplicaTracker.ReplicaState replicaState =
+ lazyWriteReplicaTracker.getNextCandidateForEviction();
+
+ while (replicaState != null &&
+ iterations++ < MAX_BLOCK_EVICTIONS_PER_ITERATION &
+ transientFreeSpaceBelowThreshold()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Evicting block " + replicaState);
+ }
+ ReplicaInfo replicaInfo = getReplicaInfo(replicaState.bpid, replicaState.blockId);
+ Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
+ File blockFile = replicaInfo.getBlockFile();
+ File metaFile = replicaInfo.getMetaFile();
+ long blockFileUsed = blockFile.length();
+ long metaFileUsed = metaFile.length();
+ lazyWriteReplicaTracker.discardReplica(replicaState, false);
+
+ // Move the replica from lazyPersist/ to finalized/ on target volume
+ BlockPoolSlice bpSlice =
+ replicaState.lazyPersistVolume.getBlockPoolSlice(replicaState.bpid);
+ File newBlockFile = bpSlice.activateSavedReplica(
+ replicaInfo, replicaState.savedBlockFile);
+
+ ReplicaInfo newReplicaInfo =
+ new FinalizedReplica(replicaInfo.getBlockId(),
+ replicaInfo.getBytesOnDisk(),
+ replicaInfo.getGenerationStamp(),
+ replicaState.lazyPersistVolume,
+ newBlockFile.getParentFile());
+
+ // Update the volumeMap entry. This removes the old entry.
+ volumeMap.add(replicaState.bpid, newReplicaInfo);
+
+ // Remove the old replicas from transient storage.
+ if (blockFile.delete() || !blockFile.exists()) {
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, blockFileUsed);
+ }
+ if (metaFile.delete() || !metaFile.exists()) {
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, metaFileUsed);
+ }
+
+ // If deletion failed then the directory scanner will cleanup the blocks
+ // eventually.
+ replicaState = lazyWriteReplicaTracker.getNextCandidateForEviction();
+ }
+ }
+
@Override
public void run() {
int numSuccessiveFailures = 0;
@@ -2459,11 +2454,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
while (fsRunning && shouldRun) {
try {
numSuccessiveFailures = saveNextReplica() ? 0 : (numSuccessiveFailures + 1);
+ evictBlocks();
// Sleep if we have no more work to do or if it looks like we are not
// making any forward progress. This is to ensure that if all persist
// operations are failing we don't keep retrying them in a tight loop.
- if (numSuccessiveFailures == lazyWriteReplicaTracker.numReplicasNotPersisted()) {
+ if (numSuccessiveFailures >= lazyWriteReplicaTracker.numReplicasNotPersisted()) {
Thread.sleep(checkpointerInterval * 1000);
numSuccessiveFailures = 0;
}
@@ -2471,7 +2467,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.info("LazyWriter was interrupted, exiting");
break;
} catch (Exception e) {
- LOG.error("Ignoring exception in LazyWriter:", e);
+ LOG.warn("Ignoring exception in LazyWriter:", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34f28eaf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
index 222b63a..9f020c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
@@ -104,30 +104,23 @@ class LazyWriteReplicaTracker {
/**
* Queue of replicas that need to be written to disk.
+ * Stale entries are GC'd by dequeueNextReplicaToPersist.
*/
final Queue<ReplicaState> replicasNotPersisted;
/**
- * A map of blockId to persist complete time for transient blocks. This allows
- * us to evict LRU blocks from transient storage. Protected by 'this'
- * Object lock.
+ * Queue of replicas in the order in which they were persisted.
+ * We'll dequeue them in the same order.
+ * We can improve the eviction scheme later.
+ * Stale entries are GC'd by getNextCandidateForEviction.
*/
- final Map<ReplicaState, Long> replicasPersisted;
+ final Queue<ReplicaState> replicasPersisted;
LazyWriteReplicaTracker(final FsDatasetImpl fsDataset) {
this.fsDataset = fsDataset;
replicaMaps = new HashMap<String, Map<Long, ReplicaState>>();
replicasNotPersisted = new LinkedList<ReplicaState>();
- replicasPersisted = new HashMap<ReplicaState, Long>();
- }
-
- TreeMultimap<Long, ReplicaState> getLruMap() {
- // TODO: This can be made more efficient.
- TreeMultimap<Long, ReplicaState> reversedMap = TreeMultimap.create();
- for (Map.Entry<ReplicaState, Long> entry : replicasPersisted.entrySet()) {
- reversedMap.put(entry.getValue(), entry.getKey());
- }
- return reversedMap;
+ replicasPersisted = new LinkedList<ReplicaState>();
}
synchronized void addReplica(String bpid, long blockId,
@@ -171,7 +164,8 @@ class LazyWriteReplicaTracker {
// one.
replicasNotPersisted.remove(replicaState);
}
- replicasPersisted.put(replicaState, System.currentTimeMillis() / 1000);
+
+ replicasPersisted.add(replicaState);
}
synchronized ReplicaState dequeueNextReplicaToPersist() {
@@ -188,14 +182,36 @@ class LazyWriteReplicaTracker {
return null;
}
- synchronized void reenqueueReplica(final ReplicaState replicaState) {
+ synchronized void reenqueueReplicaNotPersisted(final ReplicaState replicaState) {
replicasNotPersisted.add(replicaState);
}
+ synchronized void reenqueueReplicaPersisted(final ReplicaState replicaState) {
+ replicasPersisted.add(replicaState);
+ }
+
synchronized int numReplicasNotPersisted() {
return replicasNotPersisted.size();
}
+ synchronized ReplicaState getNextCandidateForEviction() {
+ while (replicasPersisted.size() != 0) {
+ ReplicaState replicaState = replicasPersisted.remove();
+ Map<Long, ReplicaState> replicaMap = replicaMaps.get(replicaState.bpid);
+
+ if (replicaMap != null && replicaMap.get(replicaState.blockId) != null) {
+ return replicaState;
+ }
+
+ // The replica no longer exists, look for the next one.
+ }
+ return null;
+ }
+
+ void discardReplica(ReplicaState replicaState, boolean force) {
+ discardReplica(replicaState.bpid, replicaState.blockId, force);
+ }
+
synchronized void discardReplica(
final String bpid, final long blockId, boolean force) {
Map<Long, ReplicaState> map = replicaMaps.get(bpid);
@@ -221,9 +237,5 @@ class LazyWriteReplicaTracker {
}
map.remove(blockId);
- replicasPersisted.remove(replicaState);
-
- // Leave the replica in replicasNotPersisted if its present.
- // dequeueNextReplicaToPersist will GC it eventually.
}
}
[16/34] git commit: HDFS-7084. FsDatasetImpl#copyBlockFiles debug log
can be improved. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-7084. FsDatasetImpl#copyBlockFiles debug log can be improved. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d32ce579
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d32ce579
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d32ce579
Branch: refs/heads/branch-2.6
Commit: d32ce5794af1adde11a1085bafae08a40d0f2958
Parents: b7062e8
Author: arp <ar...@apache.org>
Authored: Thu Sep 18 11:13:53 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:51 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32ce579/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e243852..83d94a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -724,8 +724,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
throw new IOException("Failed to copy " + srcFile + " to " + dstFile, e);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("addBlock: Moved " + srcMeta + " to " + dstMeta);
- LOG.debug("addBlock: Moved " + srcFile + " to " + dstFile);
+ LOG.debug("Copied " + srcMeta + " to " + dstMeta);
+ LOG.debug("Copied " + srcFile + " to " + dstFile);
}
return new File[] {dstMeta, dstFile};
}
[10/34] git commit: HDFS-6950. Add Additional unit tests for
HDFS-6581. (Contributed by Xiaoyu Yao)
Posted by ji...@apache.org.
HDFS-6950. Add Additional unit tests for HDFS-6581. (Contributed by Xiaoyu Yao)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4609ba0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4609ba0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4609ba0d
Branch: refs/heads/branch-2.6
Commit: 4609ba0dc899f3c488bff3ec00d48f6938facd6b
Parents: 754ba3a
Author: arp <ar...@apache.org>
Authored: Wed Sep 3 10:51:26 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:50 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 45 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 543 +++++++++++++++++--
2 files changed, 537 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4609ba0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 8c987c4..820b812 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -94,6 +94,11 @@ import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -305,16 +310,29 @@ public class DFSTestUtil {
public static void createFile(FileSystem fs, Path fileName, int bufferLen,
long fileLen, long blockSize, short replFactor, long seed)
throws IOException {
- assert bufferLen > 0;
- if (!fs.mkdirs(fileName.getParent())) {
+ createFile(fs, fileName, false, bufferLen, fileLen, blockSize,
+ replFactor, seed, false);
+ }
+
+ public static void createFile(FileSystem fs, Path fileName,
+ boolean isLazyPersist, int bufferLen, long fileLen, long blockSize,
+ short replFactor, long seed, boolean flush) throws IOException {
+ assert bufferLen > 0;
+ if (!fs.mkdirs(fileName.getParent())) {
throw new IOException("Mkdirs failed to create " +
- fileName.getParent().toString());
- }
- FSDataOutputStream out = null;
- try {
- out = fs.create(fileName, true, fs.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- replFactor, blockSize);
+ fileName.getParent().toString());
+ }
+ FSDataOutputStream out = null;
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+ createFlags.add(OVERWRITE);
+ if (isLazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+ try {
+ out = fs.create(fileName, FsPermission.getFileDefault(), createFlags,
+ fs.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ replFactor, blockSize, null);
+
if (fileLen > 0) {
byte[] toWrite = new byte[bufferLen];
Random rb = new Random(seed);
@@ -322,10 +340,13 @@ public class DFSTestUtil {
while (bytesToWrite>0) {
rb.nextBytes(toWrite);
int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen
- : (int) bytesToWrite;
+ : (int) bytesToWrite;
- out.write(toWrite, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
+ out.write(toWrite, 0, bytesToWriteNext);
+ bytesToWrite -= bytesToWriteNext;
+ }
+ if (flush) {
+ out.hsync();
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4609ba0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index cac99a7..461c44d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -16,46 +16,49 @@
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
+import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StorageType;
-import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
-import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
-import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
-
-import org.apache.log4j.Level;
-import org.junit.After;
-import org.junit.Test;
+import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
+import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
public class TestLazyPersistFiles {
public static final Log LOG = LogFactory.getLog(TestLazyPersistFiles.class);
@@ -66,8 +69,10 @@ public class TestLazyPersistFiles {
((Log4JLogger) FsDatasetImpl.LOG).getLogger().setLevel(Level.ALL);
}
+ private static final int THREADPOOL_SIZE = 10;
+
private static short REPL_FACTOR = 1;
- private static final long BLOCK_SIZE = 10485760; // 10 MB
+ private static final int BLOCK_SIZE = 10485760; // 10 MB
private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
private static final long HEARTBEAT_INTERVAL_SEC = 1;
private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
@@ -162,6 +167,26 @@ public class TestLazyPersistFiles {
}
@Test (timeout=300000)
+ public void testPlacementOnSizeLimitedRamDisk() throws IOException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK },
+ 3 * BLOCK_SIZE -1); // 2 replicas + delta
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+
+ makeTestFile(path1, BLOCK_SIZE, true);
+ makeTestFile(path2, BLOCK_SIZE, true);
+
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+ ensureFileReplicasOnStorageType(path2, RAM_DISK);
+ }
+
+ /**
+ * Client tries to write LAZY_PERSIST to same DN with no RamDisk configured
+ * Write should default to disk. No error.
+ * @throws IOException
+ */
+ @Test (timeout=300000)
public void testFallbackToDisk() throws IOException {
startUpCluster(REPL_FACTOR, null, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
@@ -172,6 +197,59 @@ public class TestLazyPersistFiles {
}
/**
+ * File can not fit in RamDisk even with eviction
+ * @throws IOException
+ */
+ @Test (timeout=300000)
+ public void testFallbackToDiskFull() throws IOException {
+ startUpCluster(REPL_FACTOR, null, BLOCK_SIZE - 1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, DEFAULT);
+ }
+
+ /**
+ * File partially fit in RamDisk after eviction.
+ * RamDisk can fit 2 blocks. Write a file with 5 blocks.
+ * Expect 2 blocks are on RamDisk whereas other 3 on disk.
+ * @throws IOException
+ */
+ @Test (timeout=300000)
+ public void testFallbackToDiskPartial()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR,
+ new StorageType[] { RAM_DISK, DEFAULT },
+ BLOCK_SIZE * 3 - 1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE * 5, true);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ triggerBlockReport();
+
+ int numBlocksOnRamDisk = 0;
+ int numBlocksOnDisk = 0;
+
+ long fileLength = client.getFileInfo(path.toString()).getLen();
+ LocatedBlocks locatedBlocks =
+ client.getLocatedBlocks(path.toString(), 0, fileLength);
+ for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+ if (locatedBlock.getStorageTypes()[0] == RAM_DISK) {
+ numBlocksOnRamDisk++;
+ }else if (locatedBlock.getStorageTypes()[0] == DEFAULT) {
+ numBlocksOnDisk++;
+ }
+ }
+ assertThat(numBlocksOnRamDisk, is(2));
+ assertThat(numBlocksOnDisk, is(3));
+ }
+
+ /**
* If the only available storage is RAM_DISK and the LAZY_PERSIST flag is not
* specified, then block placement should fail.
*
@@ -191,6 +269,10 @@ public class TestLazyPersistFiles {
}
}
+ /**
+ * Append to lazy persist file is denied.
+ * @throws IOException
+ */
@Test (timeout=300000)
public void testAppendIsDenied() throws IOException {
startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, DEFAULT }, -1);
@@ -216,7 +298,7 @@ public class TestLazyPersistFiles {
public void testLazyPersistFilesAreDiscarded()
throws IOException, InterruptedException {
startUpCluster(REPL_FACTOR,
- new StorageType[] {RAM_DISK, DEFAULT },
+ new StorageType[] { RAM_DISK, DEFAULT },
(2 * BLOCK_SIZE - 1)); // 1 replica + delta.
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -256,7 +338,7 @@ public class TestLazyPersistFiles {
@Test (timeout=300000)
public void testLazyPersistBlocksAreSaved()
throws IOException, InterruptedException {
- startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT }, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -302,8 +384,13 @@ public class TestLazyPersistFiles {
assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
}
-
- @Test (timeout=300000)
+ /**
+ * RamDisk eviction after lazy persist to disk.
+ * Evicted blocks are still readable with on-disk replicas.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
public void testRamDiskEviction()
throws IOException, InterruptedException {
startUpCluster(REPL_FACTOR,
@@ -313,7 +400,8 @@ public class TestLazyPersistFiles {
Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
- makeTestFile(path1, BLOCK_SIZE, true);
+ final int SEED = 0xFADED;
+ makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
ensureFileReplicasOnStorageType(path1, RAM_DISK);
// Sleep for a short time to allow the lazy writer thread to do its job.
@@ -323,15 +411,268 @@ public class TestLazyPersistFiles {
// Create another file with a replica on RAM_DISK.
makeTestFile(path2, BLOCK_SIZE, true);
- DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
- Thread.sleep(10 * 1000);
+ triggerBlockReport();
// Make sure that the second file's block replica is on RAM_DISK, whereas
// the original file's block replica is now on disk.
- ensureFileReplicasOnStorageType(path2, RAM_DISK);
+// ensureFileReplicasOnStorageType(path2, RAM_DISK);
ensureFileReplicasOnStorageType(path1, DEFAULT);
}
+ /**
+ * RamDisk eviction should not happen on blocks that are not yet
+ * persisted on disk.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
+ public void testRamDiskEvictionBeforePersist()
+ throws IOException, InterruptedException {
+ // 1 replica + delta, lazy persist interval every 50 minutes
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ (2 * BLOCK_SIZE - 1));
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+ final int SEED = 0XFADED;
+
+ // Stop lazy writer to ensure block for path1 is not persisted to disk.
+ stopLazyWriter(cluster.getDataNodes().get(0));
+
+ makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ // Create second file with a replica on RAM_DISK.
+ makeTestFile(path2, BLOCK_SIZE, true);
+
+ // Eviction should not happen for block of the first file that is not
+ // persisted yet.
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+ ensureFileReplicasOnStorageType(path2, DEFAULT);
+
+ assert(fs.exists(path1));
+ assert(fs.exists(path2));
+ verifyReadRandomFile(path1, BLOCK_SIZE, SEED);
+ }
+
+ /**
+ * Validates lazy persisted blocks are evicted from RAM_DISK based on LRU.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
+ public void testRamDiskEvictionLRU()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ (4 * BLOCK_SIZE -1)); // 3 replica + delta.
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final int NUM_PATHS = 6;
+ Path paths[] = new Path[NUM_PATHS];
+
+ for (int i = 0; i < NUM_PATHS; i++) {
+ paths[i] = new Path("/" + METHOD_NAME + "." + i +".dat");
+ }
+
+ // No eviction for the first half of files
+ for (int i = 0; i < NUM_PATHS/2; i++) {
+ makeTestFile(paths[i], BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
+ }
+
+ // Lazy persist writer persists the first half of files
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ // Create the second half of files with eviction upon each create.
+ for (int i = NUM_PATHS/2; i < NUM_PATHS; i++) {
+ makeTestFile(paths[i], BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
+
+ // path[i-NUM_PATHS/2] is expected to be evicted by LRU
+ triggerBlockReport();
+ ensureFileReplicasOnStorageType(paths[i - NUM_PATHS / 2], DEFAULT);
+ }
+ }
+
+ /**
+ * Delete lazy-persist file that has not been persisted to disk.
+ * Memory is freed up and file is gone.
+ * @throws IOException
+ */
+ @Test (timeout=300000)
+ public void testDeleteBeforePersist()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ stopLazyWriter(cluster.getDataNodes().get(0));
+
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, BLOCK_SIZE, true);
+ LocatedBlocks locatedBlocks =
+ ensureFileReplicasOnStorageType(path, RAM_DISK);
+
+ // Delete before persist
+ client.delete(path.toString(), false);
+ Assert.assertFalse(fs.exists(path));
+
+ assertThat(verifyDeletedBlocks(locatedBlocks), is(true));
+ }
+
+ /**
+ * Delete lazy-persist file that has been persisted to disk
+ * Both memory blocks and disk blocks are deleted.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
+ public void testDeleteAfterPersist()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT }, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE, true);
+ LocatedBlocks locatedBlocks = ensureFileReplicasOnStorageType(path, RAM_DISK);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ // Delete after persist
+ client.delete(path.toString(), false);
+ Assert.assertFalse(fs.exists(path));
+
+ triggerBlockReport();
+
+ assertThat(verifyDeletedBlocks(locatedBlocks), is(true));
+ }
+
+ /**
+ * RAM_DISK used/free space
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
+ public void testDfsUsageCreateDelete()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ 5 * BLOCK_SIZE - 1); // 4 replica + delta
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ // Get the usage before write BLOCK_SIZE
+ long usedBeforeCreate = fs.getUsed();
+
+ makeTestFile(path, BLOCK_SIZE, true);
+ long usedAfterCreate = fs.getUsed();
+
+ assertThat(usedAfterCreate, is((long) BLOCK_SIZE));
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ long usedAfterPersist = fs.getUsed();
+ assertThat(usedAfterPersist, is((long) BLOCK_SIZE));
+
+ // Delete after persist
+ client.delete(path.toString(), false);
+ long usedAfterDelete = fs.getUsed();
+
+ assertThat(usedBeforeCreate, is(usedAfterDelete));
+ }
+
+ /**
+ * Concurrent read from the same node and verify the contents.
+ */
+ @Test (timeout=300000)
+ public void testConcurrentRead()
+ throws Exception {
+ startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK },
+ 3 * BLOCK_SIZE -1); // 2 replicas + delta
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final Path path1 = new Path("/" + METHOD_NAME + ".dat");
+
+ final int SEED = 0xFADED;
+ final int NUM_TASKS = 5;
+ makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ //Read from multiple clients
+ final CountDownLatch latch = new CountDownLatch(NUM_TASKS);
+ final AtomicBoolean testFailed = new AtomicBoolean(false);
+
+ Runnable readerRunnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Assert.assertTrue(verifyReadRandomFile(path1, BLOCK_SIZE, SEED));
+ } catch (Throwable e) {
+ LOG.error("readerRunnable error", e);
+ testFailed.set(true);
+ } finally {
+ latch.countDown();
+ }
+ }
+ };
+
+ Thread threads[] = new Thread[NUM_TASKS];
+ for (int i = 0; i < NUM_TASKS; i++) {
+ threads[i] = new Thread(readerRunnable);
+ threads[i].start();
+ }
+
+ Thread.sleep(500);
+
+ for (int i = 0; i < NUM_TASKS; i++) {
+ Uninterruptibles.joinUninterruptibly(threads[i]);
+ }
+ Assert.assertFalse(testFailed.get());
+ }
+
+ /**
+ * Concurrent write with eviction
+ * RAM_DISK can hold 9 replicas
+ * 4 threads each write 5 replicas
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout=300000)
+ public void testConcurrentWrites()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
+ (10 * BLOCK_SIZE -1)); // 9 replica + delta.
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final int SEED = 0xFADED;
+ final int NUM_WRITERS = 4;
+ final int NUM_WRITER_PATHS = 5;
+
+ Path paths[][] = new Path[NUM_WRITERS][NUM_WRITER_PATHS];
+ for (int i = 0; i < NUM_WRITERS; i++) {
+ paths[i] = new Path[NUM_WRITER_PATHS];
+ for (int j = 0; j < NUM_WRITER_PATHS; j++) {
+ paths[i][j] =
+ new Path("/" + METHOD_NAME + ".Writer" + i + ".File." + j + ".dat");
+ }
+ }
+
+ final CountDownLatch latch = new CountDownLatch(NUM_WRITERS);
+ final AtomicBoolean testFailed = new AtomicBoolean(false);
+
+ ExecutorService executor = Executors.newFixedThreadPool(THREADPOOL_SIZE);
+ for (int i = 0; i < NUM_WRITERS; i++) {
+ Runnable writer = new WriterRunnable(cluster, i, paths[i], SEED, latch,
+ testFailed);
+ executor.execute(writer);
+ }
+
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+ triggerBlockReport();
+
+ // Stop executor from adding new tasks to finish existing threads in queue
+ latch.await();
+
+ assertThat(testFailed.get(), is(false));
+ }
+
@Test (timeout=300000)
public void testDnRestartWithSavedReplicas()
throws IOException, InterruptedException {
@@ -384,11 +725,12 @@ public class TestLazyPersistFiles {
/**
* If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially
- * capped. If tmpfsStorageLimit < 0 then it is ignored.
+ * capped. If ramDiskStorageLimit < 0 then it is ignored.
*/
private void startUpCluster(final int numDataNodes,
final StorageType[] storageTypes,
- final long ramDiskStorageLimit)
+ final long ramDiskStorageLimit,
+ final boolean useSCR)
throws IOException {
conf = new Configuration();
@@ -397,11 +739,13 @@ public class TestLazyPersistFiles {
LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
- HEARTBEAT_RECHECK_INTERVAL_MSEC);
+ HEARTBEAT_RECHECK_INTERVAL_MSEC);
conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
- LAZY_WRITER_INTERVAL_SEC);
+ LAZY_WRITER_INTERVAL_SEC);
+
+ conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY,useSCR);
- REPL_FACTOR = 1; //Reset if case a test has modified the value
+ REPL_FACTOR = 1; //Reset in case a test has modified the value
cluster = new MiniDFSCluster
.Builder(conf)
@@ -411,7 +755,7 @@ public class TestLazyPersistFiles {
fs = cluster.getFileSystem();
client = fs.getClient();
- // Artifically cap the storage capacity of the tmpfs volume.
+ // Artificially cap the storage capacity of the RAM_DISK volume.
if (ramDiskStorageLimit >= 0) {
List<? extends FsVolumeSpi> volumes =
cluster.getDataNodes().get(0).getFSDataset().getVolumes();
@@ -426,6 +770,13 @@ public class TestLazyPersistFiles {
LOG.info("Cluster startup complete");
}
+ private void startUpCluster(final int numDataNodes,
+ final StorageType[] storageTypes,
+ final long ramDiskStorageLimit)
+ throws IOException {
+ startUpCluster(numDataNodes, storageTypes, ramDiskStorageLimit, false);
+ }
+
private void makeTestFile(Path path, long length, final boolean isLazyPersist)
throws IOException {
@@ -435,9 +786,7 @@ public class TestLazyPersistFiles {
createFlags.add(LAZY_PERSIST);
}
-
FSDataOutputStream fos = null;
-
try {
fos =
fs.create(path,
@@ -465,13 +814,14 @@ public class TestLazyPersistFiles {
private LocatedBlocks ensureFileReplicasOnStorageType(
Path path, StorageType storageType) throws IOException {
// Ensure that returned block locations returned are correct!
+ LOG.info("Ensure path: " + path + " is on StorageType: " + storageType);
+ assertThat(fs.exists(path), is(true));
long fileLength = client.getFileInfo(path.toString()).getLen();
LocatedBlocks locatedBlocks =
client.getLocatedBlocks(path.toString(), 0, fileLength);
for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
}
-
return locatedBlocks;
}
@@ -480,4 +830,119 @@ public class TestLazyPersistFiles {
FsDatasetImpl fsDataset = ((FsDatasetImpl) dn.getFSDataset());
((FsDatasetImpl.LazyWriter) fsDataset.lazyWriter.getRunnable()).stop();
}
+
+ private void makeRandomTestFile(Path path, long length, final boolean isLazyPersist,
+ long seed) throws IOException {
+ DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
+ BLOCK_SIZE, REPL_FACTOR, seed, true);
+ }
+
+ private boolean verifyReadRandomFile(
+ Path path, int fileLength, int seed) throws IOException {
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, path);
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(seed, fileLength);
+ return Arrays.equals(contents, expected);
+ }
+
+ private boolean verifyDeletedBlocks(LocatedBlocks locatedBlocks)
+ throws IOException, InterruptedException {
+
+ LOG.info("Verifying replica has no saved copy after deletion.");
+ triggerBlockReport();
+
+ while(
+ DataNodeTestUtils.getPendingAsyncDeletions(cluster.getDataNodes().get(0))
+ > 0L){
+ Thread.sleep(1000);
+ }
+
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+
+ // Make sure deleted replica does not have a copy on either finalized dir of
+ // transient volume or finalized dir of non-transient volume
+ for (FsVolumeSpi v : volumes) {
+ FsVolumeImpl volume = (FsVolumeImpl) v;
+ File targetDir = (v.isTransientStorage()) ?
+ volume.getBlockPoolSlice(bpid).getFinalizedDir() :
+ volume.getBlockPoolSlice(bpid).getLazypersistDir();
+ if (verifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean verifyBlockDeletedFromDir(File dir, LocatedBlocks locatedBlocks) {
+
+ for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+ File targetDir =
+ DatanodeUtil.idToBlockDir(dir, lb.getBlock().getBlockId());
+
+ File blockFile = new File(targetDir, lb.getBlock().getBlockName());
+ if (blockFile.exists()) {
+ LOG.warn("blockFile: " + blockFile.getAbsolutePath() +
+ " exists after deletion.");
+ return false;
+ }
+ File metaFile = new File(targetDir,
+ DatanodeUtil.getMetaName(lb.getBlock().getBlockName(),
+ lb.getBlock().getGenerationStamp()));
+ if (metaFile.exists()) {
+ LOG.warn("metaFile: " + metaFile.getAbsolutePath() +
+ " exists after deletion.");
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void triggerBlockReport()
+ throws IOException, InterruptedException {
+ // Trigger block report to NN
+ DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
+ Thread.sleep(10 * 1000);
+ }
+
+ class WriterRunnable implements Runnable {
+ private final int id;
+ private final MiniDFSCluster cluster;
+ private final Path paths[];
+ private final int seed;
+ private CountDownLatch latch;
+ private AtomicBoolean bFail;
+
+ public WriterRunnable(MiniDFSCluster cluster, int threadIndex, Path[] paths,
+ int seed, CountDownLatch latch,
+ AtomicBoolean bFail) {
+ id = threadIndex;
+ this.cluster = cluster;
+ this.paths = paths;
+ this.seed = seed;
+ this.latch = latch;
+ this.bFail = bFail;
+ System.out.println("Creating Writer: " + id);
+ }
+
+ public void run() {
+ System.out.println("Writer " + id + " starting... ");
+ int i = 0;
+ try {
+ for (i = 0; i < paths.length; i++) {
+ makeRandomTestFile(paths[i], BLOCK_SIZE, true, seed);
+ // eviction may faiL when all blocks are not persisted yet.
+ // ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
+ }
+ } catch (IOException e) {
+ bFail.set(true);
+ LOG.error("Writer exception: writer id:" + id +
+ " testfile: " + paths[i].toString() +
+ " " + e);
+ } finally {
+ latch.countDown();
+ }
+ }
+ }
}
[26/34] git commit: HDFS-7144. Fix findbugs warnings in
RamDiskReplicaTracker. (Contributed by Tsz Wo Nicholas Sze)
Posted by ji...@apache.org.
HDFS-7144. Fix findbugs warnings in RamDiskReplicaTracker. (Contributed by Tsz Wo Nicholas Sze)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba31a950
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba31a950
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba31a950
Branch: refs/heads/branch-2.6
Commit: ba31a9503c51369a63ddaaf7af0b90c6acca7be8
Parents: 3481f5d
Author: arp <ar...@apache.org>
Authored: Thu Sep 25 11:14:24 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../impl/RamDiskReplicaLruTracker.java | 10 +++++++++
.../fsdataset/impl/RamDiskReplicaTracker.java | 22 ++++++++++++--------
2 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba31a950/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
index 0899e70..7808003 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
@@ -40,6 +40,16 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
private RamDiskReplicaLru(String bpid, long blockId, FsVolumeImpl ramDiskVolume) {
super(bpid, blockId, ramDiskVolume);
}
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return super.equals(other);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba31a950/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
index 03fc068..2401424 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import com.google.common.base.Preconditions;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +34,7 @@ import java.io.File;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public abstract class RamDiskReplicaTracker {
+ static final Log LOG = LogFactory.getLog(RamDiskReplicaTracker.class);
FsDatasetImpl fsDataset;
@@ -117,18 +121,18 @@ public abstract class RamDiskReplicaTracker {
// Delete the saved meta and block files. Failure to delete can be
// ignored, the directory scanner will retry the deletion later.
void deleteSavedFiles() {
- try {
- if (savedBlockFile != null) {
- savedBlockFile.delete();
- savedBlockFile = null;
+ if (savedBlockFile != null) {
+ if (!savedBlockFile.delete()) {
+ LOG.warn("Failed to delete block file " + savedBlockFile);
}
+ savedBlockFile = null;
+ }
- if (savedMetaFile != null) {
- savedMetaFile.delete();
- savedMetaFile = null;
+ if (savedMetaFile != null) {
+ if (!savedMetaFile.delete()) {
+ LOG.warn("Failed to delete meta file " + savedMetaFile);
}
- } catch (Throwable t) {
- // Ignore any exceptions.
+ savedMetaFile = null;
}
}
[25/34] git commit: HDFS-7100. Make eviction scheme pluggable. (Arpit
Agarwal)
Posted by ji...@apache.org.
HDFS-7100. Make eviction scheme pluggable. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9eebaa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9eebaa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9eebaa0
Branch: refs/heads/branch-2.6
Commit: e9eebaa01246dd805ef026f7f8126fe0f7eecd23
Parents: 1441988
Author: arp <ar...@apache.org>
Authored: Sat Sep 20 13:25:23 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../hadoop/hdfs/server/datanode/DataNode.java | 3 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 11 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 97 ++++---
.../datanode/fsdataset/impl/FsVolumeImpl.java | 14 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 4 +-
.../fsdataset/impl/LazyWriteReplicaTracker.java | 268 -------------------
.../impl/RamDiskReplicaLruTracker.java | 208 ++++++++++++++
.../fsdataset/impl/RamDiskReplicaTracker.java | 245 +++++++++++++++++
.../fsdataset/impl/FsDatasetTestUtil.java | 2 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 47 ++--
11 files changed, 556 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f5bba86..b5c7fed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
import org.apache.hadoop.hdfs.web.AuthFilter;
import org.apache.hadoop.http.HttpConfig;
@@ -129,6 +130,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec";
public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60;
+ public static final String DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = "dfs.datanode.ram.disk.replica.tracker";
+ public static final Class<RamDiskReplicaLruTracker> DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_DEFAULT = RamDiskReplicaLruTracker.class;
public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT = "dfs.datanode.ram.disk.low.watermark.percent";
public static final int DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT_DEFAULT = 10;
public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS = "dfs.datanode.ram.disk.low.watermark.replicas";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index bf09899..f10be62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2088,7 +2088,8 @@ public class DataNode extends ReconfigurableBase
LOG.warn("Cannot find BPOfferService for reporting block received for bpid="
+ block.getBlockPoolId());
}
- if (blockScanner != null) {
+ FsVolumeSpi volume = getFSDataset().getVolume(block);
+ if (blockScanner != null && !volume.isTransientStorage()) {
blockScanner.addBlock(block);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 06d60b1..bfa1772 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -292,9 +292,9 @@ class BlockPoolSlice {
* Move a persisted replica from lazypersist directory to a subdirectory
* under finalized.
*/
- File activateSavedReplica(Block b, File blockFile) throws IOException {
+ File activateSavedReplica(Block b, File metaFile, File blockFile)
+ throws IOException {
final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId());
- final File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
final File targetBlockFile = new File(blockDir, blockFile.getName());
final File targetMetaFile = new File(blockDir, metaFile.getName());
FileUtils.moveFile(blockFile, targetBlockFile);
@@ -313,7 +313,7 @@ class BlockPoolSlice {
void getVolumeMap(ReplicaMap volumeMap,
- final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ final RamDiskReplicaTracker lazyWriteReplicaMap)
throws IOException {
// Recover lazy persist replicas, they will be added to the volumeMap
// when we scan the finalized directory.
@@ -410,7 +410,7 @@ class BlockPoolSlice {
* false if the directory has rbw replicas
*/
void addToReplicasMap(ReplicaMap volumeMap, File dir,
- final LazyWriteReplicaTracker lazyWriteReplicaMap,
+ final RamDiskReplicaTracker lazyWriteReplicaMap,
boolean isFinalized)
throws IOException {
File files[] = FileUtil.listFiles(dir);
@@ -487,7 +487,8 @@ class BlockPoolSlice {
// it is in the lazyWriteReplicaMap so it can be persisted
// eventually.
if (newReplica.getVolume().isTransientStorage()) {
- lazyWriteReplicaMap.addReplica(bpid, blockId, newReplica.getVolume());
+ lazyWriteReplicaMap.addReplica(bpid, blockId,
+ (FsVolumeImpl) newReplica.getVolume());
} else {
lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 83d94a3..11322a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -30,7 +30,6 @@ import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -88,6 +87,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica;
import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -159,7 +159,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public synchronized Block getStoredBlock(String bpid, long blkid)
throws IOException {
- File blockfile = getFile(bpid, blkid);
+ File blockfile = getFile(bpid, blkid, false);
if (blockfile == null) {
return null;
}
@@ -219,7 +219,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
private volatile boolean fsRunning;
final ReplicaMap volumeMap;
- final LazyWriteReplicaTracker lazyWriteReplicaTracker;
+ final RamDiskReplicaTracker ramDiskReplicaTracker;
private static final int MAX_BLOCK_EVICTIONS_PER_ITERATION = 3;
@@ -263,7 +263,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
storageMap = new ConcurrentHashMap<String, DatanodeStorage>();
volumeMap = new ReplicaMap(this);
- lazyWriteReplicaTracker = new LazyWriteReplicaTracker(this);
+ ramDiskReplicaTracker = RamDiskReplicaTracker.getInstance(conf, this);
@SuppressWarnings("unchecked")
final VolumeChoosingPolicy<FsVolumeImpl> blockChooserImpl =
@@ -298,7 +298,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
FsVolumeImpl fsVolume = FsVolumeImplAllocator.createVolume(
this, sd.getStorageUuid(), dir, this.conf, storageType);
ReplicaMap tempVolumeMap = new ReplicaMap(this);
- fsVolume.getVolumeMap(volumeMap, lazyWriteReplicaTracker);
+ fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker);
volumeMap.addAll(tempVolumeMap);
volumes.addVolume(fsVolume);
@@ -326,7 +326,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
for (final String bpid : bpids) {
try {
fsVolume.addBlockPool(bpid, this.conf);
- fsVolume.getVolumeMap(bpid, tempVolumeMap);
+ fsVolume.getVolumeMap(bpid, tempVolumeMap, ramDiskReplicaTracker);
} catch (IOException e) {
LOG.warn("Caught exception when adding " + fsVolume +
". Will throw later.", e);
@@ -586,12 +586,16 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
* checking that it exists. This should be used when the
* next operation is going to open the file for read anyway,
* and thus the exists check is redundant.
+ *
+ * @param touch if true then update the last access timestamp of the
+ * block. Currently used for blocks on transient storage.
*/
- private File getBlockFileNoExistsCheck(ExtendedBlock b)
+ private File getBlockFileNoExistsCheck(ExtendedBlock b,
+ boolean touch)
throws IOException {
final File f;
synchronized(this) {
- f = getFile(b.getBlockPoolId(), b.getLocalBlock().getBlockId());
+ f = getFile(b.getBlockPoolId(), b.getLocalBlock().getBlockId(), touch);
}
if (f == null) {
throw new IOException("Block " + b + " is not valid");
@@ -602,7 +606,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public InputStream getBlockInputStream(ExtendedBlock b,
long seekOffset) throws IOException {
- File blockFile = getBlockFileNoExistsCheck(b);
+ File blockFile = getBlockFileNoExistsCheck(b, true);
if (isNativeIOAvailable) {
return NativeIO.getShareDeleteFileInputStream(blockFile, seekOffset);
} else {
@@ -1240,7 +1244,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile());
if (v.isTransientStorage()) {
- lazyWriteReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
+ ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
}
}
volumeMap.add(bpid, newReplicaInfo);
@@ -1265,7 +1269,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.warn("Block " + b + " unfinalized and removed. " );
}
if (replicaInfo.getVolume().isTransientStorage()) {
- lazyWriteReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true);
+ ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true);
}
}
}
@@ -1411,7 +1415,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
//Should we check for metadata file too?
final File f;
synchronized(this) {
- f = getFile(bpid, blockId);
+ f = getFile(bpid, blockId, false);
}
if(f != null ) {
@@ -1496,7 +1500,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
if (v.isTransientStorage()) {
- lazyWriteReplicaTracker.discardReplica(bpid, invalidBlks[i].getBlockId(), true);
+ ramDiskReplicaTracker.discardReplica(bpid, invalidBlks[i].getBlockId(), true);
}
// If a DFSClient has the replica in its cache of short-circuit file
@@ -1628,7 +1632,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public synchronized boolean contains(final ExtendedBlock block) {
final long blockId = block.getLocalBlock().getBlockId();
- return getFile(block.getBlockPoolId(), blockId) != null;
+ return getFile(block.getBlockPoolId(), blockId, false) != null;
}
/**
@@ -1637,9 +1641,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
* @param blockId a block's id
* @return on disk data file path; null if the replica does not exist
*/
- File getFile(final String bpid, final long blockId) {
+ File getFile(final String bpid, final long blockId, boolean touch) {
ReplicaInfo info = volumeMap.get(bpid, blockId);
if (info != null) {
+ if (touch && info.getVolume().isTransientStorage()) {
+ ramDiskReplicaTracker.touch(bpid, blockId);
+ }
return info.getBlockFile();
}
return null;
@@ -1808,7 +1815,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
blockScanner.deleteBlock(bpid, new Block(blockId));
}
if (vol.isTransientStorage()) {
- lazyWriteReplicaTracker.discardReplica(bpid, blockId, true);
+ ramDiskReplicaTracker.discardReplica(bpid, blockId, true);
}
LOG.warn("Removed block " + blockId
+ " from memory with missing block file on the disk");
@@ -1830,11 +1837,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
diskFile.length(), diskGS, vol, diskFile.getParentFile());
volumeMap.add(bpid, diskBlockInfo);
final DataBlockScanner blockScanner = datanode.getBlockScanner();
- if (blockScanner != null) {
- blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo));
- }
- if (vol.isTransientStorage()) {
- lazyWriteReplicaTracker.addReplica(bpid, blockId, (FsVolumeImpl) vol);
+ if (!vol.isTransientStorage()) {
+ if (blockScanner != null) {
+ blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo));
+ }
+ } else {
+ ramDiskReplicaTracker.addReplica(bpid, blockId, (FsVolumeImpl) vol);
}
LOG.warn("Added missing block to memory " + diskBlockInfo);
return;
@@ -2117,7 +2125,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumes.addBlockPool(bpid, conf);
volumeMap.initBlockPool(bpid);
}
- volumes.getAllVolumesMap(bpid, volumeMap, lazyWriteReplicaTracker);
+ volumes.getAllVolumesMap(bpid, volumeMap, ramDiskReplicaTracker);
}
@Override
@@ -2347,7 +2355,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
LOG.debug("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
}
- lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
+ ramDiskReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
bpSlice = targetVolume.getBlockPoolSlice(bpid);
srcMeta = replicaInfo.getMetaFile();
srcFile = replicaInfo.getBlockFile();
@@ -2359,7 +2367,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
bpSlice.lazyPersistReplica(blockId, genStamp, srcMeta, srcFile);
synchronized (FsDatasetImpl.this) {
- lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedFiles);
+ ramDiskReplicaTracker.recordEndLazyPersist(bpid, blockId, savedFiles);
if (LOG.isDebugEnabled()) {
LOG.debug("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
@@ -2374,21 +2382,21 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
* @return true if there is more work to be done, false otherwise.
*/
private boolean saveNextReplica() {
- LazyWriteReplicaTracker.ReplicaState replicaState = null;
+ RamDiskReplica block = null;
boolean succeeded = false;
try {
- replicaState = lazyWriteReplicaTracker.dequeueNextReplicaToPersist();
- if (replicaState != null) {
- moveReplicaToNewVolume(replicaState.bpid, replicaState.blockId);
+ block = ramDiskReplicaTracker.dequeueNextReplicaToPersist();
+ if (block != null) {
+ moveReplicaToNewVolume(block.getBlockPoolId(), block.getBlockId());
}
succeeded = true;
} catch(IOException ioe) {
- LOG.warn("Exception saving replica " + replicaState, ioe);
+ LOG.warn("Exception saving replica " + block, ioe);
} finally {
- if (!succeeded && replicaState != null) {
- LOG.warn("Failed to save replica " + replicaState + ". re-enqueueing it.");
- lazyWriteReplicaTracker.reenqueueReplicaNotPersisted(replicaState);
+ if (!succeeded && block != null) {
+ LOG.warn("Failed to save replica " + block + ". re-enqueueing it.");
+ ramDiskReplicaTracker.reenqueueReplicaNotPersisted(block);
}
}
@@ -2426,8 +2434,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
while (iterations++ < MAX_BLOCK_EVICTIONS_PER_ITERATION &&
transientFreeSpaceBelowThreshold()) {
- LazyWriteReplicaTracker.ReplicaState replicaState =
- lazyWriteReplicaTracker.getNextCandidateForEviction();
+ RamDiskReplica replicaState = ramDiskReplicaTracker.getNextCandidateForEviction();
if (replicaState == null) {
break;
@@ -2440,46 +2447,48 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaInfo replicaInfo, newReplicaInfo;
File blockFile, metaFile;
long blockFileUsed, metaFileUsed;
+ final String bpid = replicaState.getBlockPoolId();
synchronized (FsDatasetImpl.this) {
- replicaInfo = getReplicaInfo(replicaState.bpid, replicaState.blockId);
+ replicaInfo = getReplicaInfo(replicaState.getBlockPoolId(), replicaState.getBlockId());
Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
blockFile = replicaInfo.getBlockFile();
metaFile = replicaInfo.getMetaFile();
blockFileUsed = blockFile.length();
metaFileUsed = metaFile.length();
- lazyWriteReplicaTracker.discardReplica(replicaState, false);
+ ramDiskReplicaTracker.discardReplica(replicaState, false);
// Move the replica from lazyPersist/ to finalized/ on target volume
BlockPoolSlice bpSlice =
- replicaState.lazyPersistVolume.getBlockPoolSlice(replicaState.bpid);
+ replicaState.getLazyPersistVolume().getBlockPoolSlice(bpid);
File newBlockFile = bpSlice.activateSavedReplica(
- replicaInfo, replicaState.savedBlockFile);
+ replicaInfo, replicaState.getSavedMetaFile(),
+ replicaState.getSavedBlockFile());
newReplicaInfo =
new FinalizedReplica(replicaInfo.getBlockId(),
replicaInfo.getBytesOnDisk(),
replicaInfo.getGenerationStamp(),
- replicaState.lazyPersistVolume,
+ replicaState.getLazyPersistVolume(),
newBlockFile.getParentFile());
// Update the volumeMap entry.
- volumeMap.add(replicaState.bpid, newReplicaInfo);
+ volumeMap.add(bpid, newReplicaInfo);
}
// Before deleting the files from transient storage we must notify the
// NN that the files are on the new storage. Else a blockReport from
// the transient storage might cause the NN to think the blocks are lost.
ExtendedBlock extendedBlock =
- new ExtendedBlock(replicaState.bpid, newReplicaInfo);
+ new ExtendedBlock(bpid, newReplicaInfo);
datanode.notifyNamenodeReceivedBlock(
extendedBlock, null, newReplicaInfo.getStorageUuid());
// Remove the old replicas from transient storage.
if (blockFile.delete() || !blockFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, blockFileUsed);
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed);
if (metaFile.delete() || !metaFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(replicaState.bpid, metaFileUsed);
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed);
}
}
@@ -2500,7 +2509,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Sleep if we have no more work to do or if it looks like we are not
// making any forward progress. This is to ensure that if all persist
// operations are failing we don't keep retrying them in a tight loop.
- if (numSuccessiveFailures >= lazyWriteReplicaTracker.numReplicasNotPersisted()) {
+ if (numSuccessiveFailures >= ramDiskReplicaTracker.numReplicasNotPersisted()) {
Thread.sleep(checkpointerInterval * 1000);
numSuccessiveFailures = 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index e2efb2f..098ad09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -235,9 +235,6 @@ public class FsVolumeImpl implements FsVolumeSpi {
@Override
public void reserveSpaceForRbw(long bytesToReserve) {
if (bytesToReserve != 0) {
- if (FsDatasetImpl.LOG.isDebugEnabled()) {
- FsDatasetImpl.LOG.debug("Reserving " + bytesToReserve + " on volume " + getBasePath());
- }
reservedForRbw.addAndGet(bytesToReserve);
}
}
@@ -245,9 +242,6 @@ public class FsVolumeImpl implements FsVolumeSpi {
@Override
public void releaseReservedSpace(long bytesToRelease) {
if (bytesToRelease != 0) {
- if (FsDatasetImpl.LOG.isDebugEnabled()) {
- FsDatasetImpl.LOG.debug("Releasing " + bytesToRelease + " on volume " + getBasePath());
- }
long oldReservation, newReservation;
do {
@@ -298,17 +292,17 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
void getVolumeMap(ReplicaMap volumeMap,
- final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ final RamDiskReplicaTracker ramDiskReplicaMap)
throws IOException {
for(BlockPoolSlice s : bpSlices.values()) {
- s.getVolumeMap(volumeMap, lazyWriteReplicaMap);
+ s.getVolumeMap(volumeMap, ramDiskReplicaMap);
}
}
void getVolumeMap(String bpid, ReplicaMap volumeMap,
- final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ final RamDiskReplicaTracker ramDiskReplicaMap)
throws IOException {
- getBlockPoolSlice(bpid).getVolumeMap(volumeMap, lazyWriteReplicaMap);
+ getBlockPoolSlice(bpid).getVolumeMap(volumeMap, ramDiskReplicaMap);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 82fe35f..837ddf7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -121,7 +121,7 @@ class FsVolumeList {
void getAllVolumesMap(final String bpid,
final ReplicaMap volumeMap,
- final LazyWriteReplicaTracker lazyWriteReplicaMap)
+ final RamDiskReplicaTracker ramDiskReplicaMap)
throws IOException {
long totalStartTime = Time.monotonicNow();
final List<IOException> exceptions = Collections.synchronizedList(
@@ -134,7 +134,7 @@ class FsVolumeList {
FsDatasetImpl.LOG.info("Adding replicas to map for block pool " +
bpid + " on volume " + v + "...");
long startTime = Time.monotonicNow();
- v.getVolumeMap(bpid, volumeMap, lazyWriteReplicaMap);
+ v.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap);
long timeTaken = Time.monotonicNow() - startTime;
FsDatasetImpl.LOG.info("Time to add replicas to map for block pool"
+ " " + bpid + " on volume " + v + ": " + timeTaken + "ms");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
deleted file mode 100644
index e8d9c5c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-
-
-import com.google.common.collect.TreeMultimap;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-
-import java.io.File;
-import java.util.*;
-
-class LazyWriteReplicaTracker {
-
- enum State {
- IN_MEMORY,
- LAZY_PERSIST_IN_PROGRESS,
- LAZY_PERSIST_COMPLETE,
- }
-
- static class ReplicaState implements Comparable<ReplicaState> {
-
- final String bpid;
- final long blockId;
- State state;
-
- /**
- * transient storage volume that holds the original replica.
- */
- final FsVolumeSpi transientVolume;
-
- /**
- * Persistent volume that holds or will hold the saved replica.
- */
- FsVolumeImpl lazyPersistVolume;
- File savedMetaFile;
- File savedBlockFile;
-
- ReplicaState(final String bpid, final long blockId, FsVolumeSpi transientVolume) {
- this.bpid = bpid;
- this.blockId = blockId;
- this.transientVolume = transientVolume;
- state = State.IN_MEMORY;
- lazyPersistVolume = null;
- savedMetaFile = null;
- savedBlockFile = null;
- }
-
- void deleteSavedFiles() {
- try {
- if (savedBlockFile != null) {
- savedBlockFile.delete();
- savedBlockFile = null;
- }
-
- if (savedMetaFile != null) {
- savedMetaFile.delete();
- savedMetaFile = null;
- }
- } catch (Throwable t) {
- // Ignore any exceptions.
- }
- }
-
- @Override
- public String toString() {
- return "[Bpid=" + bpid + ";blockId=" + blockId + "]";
- }
-
- @Override
- public int hashCode() {
- return bpid.hashCode() ^ (int) blockId;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
-
- if (other == null || getClass() != other.getClass()) {
- return false;
- }
-
- ReplicaState otherState = (ReplicaState) other;
- return (otherState.bpid.equals(bpid) && otherState.blockId == blockId);
- }
-
- @Override
- public int compareTo(ReplicaState other) {
- if (blockId == other.blockId) {
- return 0;
- } else if (blockId < other.blockId) {
- return -1;
- } else {
- return 1;
- }
- }
- }
-
- final FsDatasetImpl fsDataset;
-
- /**
- * Map of blockpool ID to map of blockID to ReplicaInfo.
- */
- final Map<String, Map<Long, ReplicaState>> replicaMaps;
-
- /**
- * Queue of replicas that need to be written to disk.
- * Stale entries are GC'd by dequeueNextReplicaToPersist.
- */
- final Queue<ReplicaState> replicasNotPersisted;
-
- /**
- * Queue of replicas in the order in which they were persisted.
- * We'll dequeue them in the same order.
- * We can improve the eviction scheme later.
- * Stale entries are GC'd by getNextCandidateForEviction.
- */
- final Queue<ReplicaState> replicasPersisted;
-
- LazyWriteReplicaTracker(final FsDatasetImpl fsDataset) {
- this.fsDataset = fsDataset;
- replicaMaps = new HashMap<String, Map<Long, ReplicaState>>();
- replicasNotPersisted = new LinkedList<ReplicaState>();
- replicasPersisted = new LinkedList<ReplicaState>();
- }
-
- synchronized void addReplica(String bpid, long blockId,
- final FsVolumeSpi transientVolume) {
- Map<Long, ReplicaState> map = replicaMaps.get(bpid);
- if (map == null) {
- map = new HashMap<Long, ReplicaState>();
- replicaMaps.put(bpid, map);
- }
- ReplicaState replicaState = new ReplicaState(bpid, blockId, transientVolume);
- map.put(blockId, replicaState);
- replicasNotPersisted.add(replicaState);
- }
-
- synchronized void recordStartLazyPersist(
- final String bpid, final long blockId, FsVolumeImpl checkpointVolume) {
- Map<Long, ReplicaState> map = replicaMaps.get(bpid);
- ReplicaState replicaState = map.get(blockId);
- replicaState.state = State.LAZY_PERSIST_IN_PROGRESS;
- replicaState.lazyPersistVolume = checkpointVolume;
- }
-
- /**
- * @param bpid
- * @param blockId
- * @param savedFiles The saved meta and block files, in that order.
- */
- synchronized void recordEndLazyPersist(
- final String bpid, final long blockId, final File[] savedFiles) {
- Map<Long, ReplicaState> map = replicaMaps.get(bpid);
- ReplicaState replicaState = map.get(blockId);
-
- if (replicaState == null) {
- throw new IllegalStateException("Unknown replica bpid=" +
- bpid + "; blockId=" + blockId);
- }
- replicaState.state = State.LAZY_PERSIST_COMPLETE;
- replicaState.savedMetaFile = savedFiles[0];
- replicaState.savedBlockFile = savedFiles[1];
-
- if (replicasNotPersisted.peek() == replicaState) {
- // Common case.
- replicasNotPersisted.remove();
- } else {
- // Should never occur in practice as lazy writer always persists
- // the replica at the head of the queue before moving to the next
- // one.
- replicasNotPersisted.remove(replicaState);
- }
-
- replicasPersisted.add(replicaState);
- }
-
- synchronized ReplicaState dequeueNextReplicaToPersist() {
- while (replicasNotPersisted.size() != 0) {
- ReplicaState replicaState = replicasNotPersisted.remove();
- Map<Long, ReplicaState> replicaMap = replicaMaps.get(replicaState.bpid);
-
- if (replicaMap != null && replicaMap.get(replicaState.blockId) != null) {
- return replicaState;
- }
-
- // The replica no longer exists, look for the next one.
- }
- return null;
- }
-
- synchronized void reenqueueReplicaNotPersisted(final ReplicaState replicaState) {
- replicasNotPersisted.add(replicaState);
- }
-
- synchronized void reenqueueReplicaPersisted(final ReplicaState replicaState) {
- replicasPersisted.add(replicaState);
- }
-
- synchronized int numReplicasNotPersisted() {
- return replicasNotPersisted.size();
- }
-
- synchronized ReplicaState getNextCandidateForEviction() {
- while (replicasPersisted.size() != 0) {
- ReplicaState replicaState = replicasPersisted.remove();
- Map<Long, ReplicaState> replicaMap = replicaMaps.get(replicaState.bpid);
-
- if (replicaMap != null && replicaMap.get(replicaState.blockId) != null) {
- return replicaState;
- }
-
- // The replica no longer exists, look for the next one.
- }
- return null;
- }
-
- void discardReplica(ReplicaState replicaState, boolean deleteSavedCopies) {
- discardReplica(replicaState.bpid, replicaState.blockId, deleteSavedCopies);
- }
-
- /**
- * Discard any state we are tracking for the given replica. This could mean
- * the block is either deleted from the block space or the replica is no longer
- * on transient storage.
- *
- * @param deleteSavedCopies true if we should delete the saved copies on
- * persistent storage. This should be set by the
- * caller when the block is no longer needed.
- */
- synchronized void discardReplica(
- final String bpid, final long blockId,
- boolean deleteSavedCopies) {
- Map<Long, ReplicaState> map = replicaMaps.get(bpid);
-
- if (map == null) {
- return;
- }
-
- ReplicaState replicaState = map.get(blockId);
-
- if (replicaState == null) {
- return;
- }
-
- if (deleteSavedCopies) {
- replicaState.deleteSavedFiles();
- }
- map.remove(blockId);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
new file mode 100644
index 0000000..0899e70
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+
+import com.google.common.collect.TreeMultimap;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.File;
+import java.util.*;
+
+/**
+ * An implementation of RamDiskReplicaTracker that uses an LRU
+ * eviction scheme.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
+
+ private class RamDiskReplicaLru extends RamDiskReplica {
+ long lastUsedTime;
+
+ private RamDiskReplicaLru(String bpid, long blockId, FsVolumeImpl ramDiskVolume) {
+ super(bpid, blockId, ramDiskVolume);
+ }
+ }
+
+ /**
+ * Map of blockpool ID to <map of blockID to ReplicaInfo>.
+ */
+ Map<String, Map<Long, RamDiskReplicaLru>> replicaMaps;
+
+ /**
+ * Queue of replicas that need to be written to disk.
+ * Stale entries are GC'd by dequeueNextReplicaToPersist.
+ */
+ Queue<RamDiskReplicaLru> replicasNotPersisted;
+
+ /**
+ * Map of persisted replicas ordered by their last use times.
+ */
+ TreeMultimap<Long, RamDiskReplicaLru> replicasPersisted;
+
+ RamDiskReplicaLruTracker() {
+ replicaMaps = new HashMap<String, Map<Long, RamDiskReplicaLru>>();
+ replicasNotPersisted = new LinkedList<RamDiskReplicaLru>();
+ replicasPersisted = TreeMultimap.create();
+ }
+
+ @Override
+ synchronized void addReplica(final String bpid, final long blockId,
+ final FsVolumeImpl transientVolume) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+ if (map == null) {
+ map = new HashMap<Long, RamDiskReplicaLru>();
+ replicaMaps.put(bpid, map);
+ }
+ RamDiskReplicaLru ramDiskReplicaLru = new RamDiskReplicaLru(bpid, blockId, transientVolume);
+ map.put(blockId, ramDiskReplicaLru);
+ replicasNotPersisted.add(ramDiskReplicaLru);
+ }
+
+ @Override
+ synchronized void touch(final String bpid,
+ final long blockId) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+ RamDiskReplicaLru ramDiskReplicaLru = map.get(blockId);
+
+ if (ramDiskReplicaLru == null) {
+ return;
+ }
+
+ // Reinsert the replica with its new timestamp.
+ if (replicasPersisted.remove(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru)) {
+ ramDiskReplicaLru.lastUsedTime = System.currentTimeMillis();
+ replicasPersisted.put(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru);
+ }
+ }
+
+ @Override
+ synchronized void recordStartLazyPersist(
+ final String bpid, final long blockId, FsVolumeImpl checkpointVolume) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+ RamDiskReplicaLru ramDiskReplicaLru = map.get(blockId);
+ ramDiskReplicaLru.setLazyPersistVolume(checkpointVolume);
+ }
+
+ @Override
+ synchronized void recordEndLazyPersist(
+ final String bpid, final long blockId, final File[] savedFiles) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+ RamDiskReplicaLru ramDiskReplicaLru = map.get(blockId);
+
+ if (ramDiskReplicaLru == null) {
+ throw new IllegalStateException("Unknown replica bpid=" +
+ bpid + "; blockId=" + blockId);
+ }
+ ramDiskReplicaLru.recordSavedBlockFiles(savedFiles);
+
+ if (replicasNotPersisted.peek() == ramDiskReplicaLru) {
+ // Common case.
+ replicasNotPersisted.remove();
+ } else {
+ // Caller error? Fallback to O(n) removal.
+ replicasNotPersisted.remove(ramDiskReplicaLru);
+ }
+
+ ramDiskReplicaLru.lastUsedTime = System.currentTimeMillis();
+ replicasPersisted.put(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru);
+ }
+
+ @Override
+ synchronized RamDiskReplicaLru dequeueNextReplicaToPersist() {
+ while (replicasNotPersisted.size() != 0) {
+ RamDiskReplicaLru ramDiskReplicaLru = replicasNotPersisted.remove();
+ Map<Long, RamDiskReplicaLru> replicaMap =
+ replicaMaps.get(ramDiskReplicaLru.getBlockPoolId());
+
+ if (replicaMap != null && replicaMap.get(ramDiskReplicaLru.getBlockId()) != null) {
+ return ramDiskReplicaLru;
+ }
+
+ // The replica no longer exists, look for the next one.
+ }
+ return null;
+ }
+
+ @Override
+ synchronized void reenqueueReplicaNotPersisted(final RamDiskReplica ramDiskReplicaLru) {
+ replicasNotPersisted.add((RamDiskReplicaLru) ramDiskReplicaLru);
+ }
+
+ @Override
+ synchronized int numReplicasNotPersisted() {
+ return replicasNotPersisted.size();
+ }
+
+ @Override
+ synchronized RamDiskReplicaLru getNextCandidateForEviction() {
+ Iterator it = replicasPersisted.values().iterator();
+ while (it.hasNext()) {
+ RamDiskReplicaLru ramDiskReplicaLru = (RamDiskReplicaLru) it.next();
+ it.remove();
+
+ Map<Long, RamDiskReplicaLru> replicaMap =
+ replicaMaps.get(ramDiskReplicaLru.getBlockPoolId());
+
+ if (replicaMap != null && replicaMap.get(ramDiskReplicaLru.getBlockId()) != null) {
+ return ramDiskReplicaLru;
+ }
+
+ // The replica no longer exists, look for the next one.
+ }
+ return null;
+ }
+
+ /**
+ * Discard any state we are tracking for the given replica. This could mean
+ * the block is either deleted from the block space or the replica is no longer
+ * on transient storage.
+ *
+ * @param deleteSavedCopies true if we should delete the saved copies on
+ * persistent storage. This should be set by the
+ * caller when the block is no longer needed.
+ */
+ @Override
+ synchronized void discardReplica(
+ final String bpid, final long blockId,
+ boolean deleteSavedCopies) {
+ Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
+
+ if (map == null) {
+ return;
+ }
+
+ RamDiskReplicaLru ramDiskReplicaLru = map.get(blockId);
+
+ if (ramDiskReplicaLru == null) {
+ return;
+ }
+
+ if (deleteSavedCopies) {
+ ramDiskReplicaLru.deleteSavedFiles();
+ }
+
+ map.remove(blockId);
+ replicasPersisted.remove(ramDiskReplicaLru.lastUsedTime, ramDiskReplicaLru);
+
+ // replicasNotPersisted will be lazily GC'ed.
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
new file mode 100644
index 0000000..03fc068
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.File;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class RamDiskReplicaTracker {
+
+ FsDatasetImpl fsDataset;
+
+ static class RamDiskReplica implements Comparable<RamDiskReplica> {
+ private final String bpid;
+ private final long blockId;
+ private File savedBlockFile;
+ private File savedMetaFile;
+
+ /**
+ * RAM_DISK volume that holds the original replica.
+ */
+ final FsVolumeSpi ramDiskVolume;
+
+ /**
+ * Persistent volume that holds or will hold the saved replica.
+ */
+ FsVolumeImpl lazyPersistVolume;
+
+ RamDiskReplica(final String bpid, final long blockId,
+ final FsVolumeImpl ramDiskVolume) {
+ this.bpid = bpid;
+ this.blockId = blockId;
+ this.ramDiskVolume = ramDiskVolume;
+ lazyPersistVolume = null;
+ savedMetaFile = null;
+ savedBlockFile = null;
+ }
+
+ long getBlockId() {
+ return blockId;
+ }
+
+ String getBlockPoolId() {
+ return bpid;
+ }
+
+ FsVolumeImpl getLazyPersistVolume() {
+ return lazyPersistVolume;
+ }
+
+ void setLazyPersistVolume(FsVolumeImpl volume) {
+ Preconditions.checkState(!volume.isTransientStorage());
+ this.lazyPersistVolume = volume;
+ }
+
+ File getSavedBlockFile() {
+ return savedBlockFile;
+ }
+
+ File getSavedMetaFile() {
+ return savedMetaFile;
+ }
+
+ /**
+ * Record the saved meta and block files on the given volume.
+ *
+ * @param files Meta and block files, in that order.
+ */
+ void recordSavedBlockFiles(File[] files) {
+ this.savedMetaFile = files[0];
+ this.savedBlockFile = files[1];
+ }
+
+ @Override
+ public int hashCode() {
+ return bpid.hashCode() ^ (int) blockId;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ RamDiskReplica otherState = (RamDiskReplica) other;
+ return (otherState.bpid.equals(bpid) && otherState.blockId == blockId);
+ }
+
+ // Delete the saved meta and block files. Failure to delete can be
+ // ignored, the directory scanner will retry the deletion later.
+ void deleteSavedFiles() {
+ try {
+ if (savedBlockFile != null) {
+ savedBlockFile.delete();
+ savedBlockFile = null;
+ }
+
+ if (savedMetaFile != null) {
+ savedMetaFile.delete();
+ savedMetaFile = null;
+ }
+ } catch (Throwable t) {
+ // Ignore any exceptions.
+ }
+ }
+
+ @Override
+ public int compareTo(RamDiskReplica other) {
+ int bpidResult = bpid.compareTo(other.bpid);
+ if (bpidResult == 0)
+ if (blockId == other.blockId) {
+ return 0;
+ } else if (blockId < other.blockId) {
+ return -1;
+ } else {
+ return 1;
+ }
+ return bpidResult;
+ }
+
+ @Override
+ public String toString() {
+ return "[BlockPoolID=" + bpid + "; BlockId=" + blockId + "]";
+ }
+ }
+
+ /**
+ * Get an instance of the configured RamDiskReplicaTracker based on the
+ * the configuration property
+ * {@link org.apache.hadoop.hdfs.DFSConfigKeys#DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY}.
+ *
+ * @param conf the configuration to be used
+ * @param dataset the FsDataset object.
+ * @return an instance of RamDiskReplicaTracker
+ */
+ static RamDiskReplicaTracker getInstance(final Configuration conf,
+ final FsDatasetImpl fsDataset) {
+ final Class<? extends RamDiskReplicaTracker> trackerClass = conf.getClass(
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY,
+ DFSConfigKeys.DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_DEFAULT,
+ RamDiskReplicaTracker.class);
+ final RamDiskReplicaTracker tracker = ReflectionUtils.newInstance(
+ trackerClass, conf);
+ tracker.initialize(fsDataset);
+ return tracker;
+ }
+
+ void initialize(final FsDatasetImpl fsDataset) {
+ this.fsDataset = fsDataset;
+ }
+
+ /**
+ * Start tracking a new finalized replica on RAM disk.
+ *
+ * @param transientVolume RAM disk volume that stores the replica.
+ */
+ abstract void addReplica(final String bpid, final long blockId,
+ final FsVolumeImpl transientVolume);
+
+ /**
+ * Invoked when a replica is opened by a client. This may be used as
+ * a heuristic by the eviction scheme.
+ */
+ abstract void touch(final String bpid, final long blockId);
+
+ /**
+ * Get the next replica to write to persistent storage.
+ */
+ abstract RamDiskReplica dequeueNextReplicaToPersist();
+
+ /**
+ * Invoked if a replica that was previously dequeued for persistence
+ * could not be successfully persisted. Add it back so it can be retried
+ * later.
+ */
+ abstract void reenqueueReplicaNotPersisted(
+ final RamDiskReplica ramDiskReplica);
+
+ /**
+ * Invoked when the Lazy persist operation is started by the DataNode.
+ * @param checkpointVolume
+ */
+ abstract void recordStartLazyPersist(
+ final String bpid, final long blockId, FsVolumeImpl checkpointVolume);
+
+ /**
+ * Invoked when the Lazy persist operation is complete.
+ *
+ * @param savedFiles The saved meta and block files, in that order.
+ */
+ abstract void recordEndLazyPersist(
+ final String bpid, final long blockId, final File[] savedFiles);
+
+ /**
+ * Return a candidate replica to remove from RAM Disk. The exact replica
+ * to be returned may depend on the eviction scheme utilized.
+ *
+ * @return
+ */
+ abstract RamDiskReplica getNextCandidateForEviction();
+
+ /**
+ * Return the number of replicas pending persistence to disk.
+ */
+ abstract int numReplicasNotPersisted();
+
+ /**
+ * Discard all state we are tracking for the given replica.
+ */
+ abstract void discardReplica(
+ final String bpid, final long blockId,
+ boolean deleteSavedCopies);
+
+ void discardReplica(RamDiskReplica replica, boolean deleteSavedCopies) {
+ discardReplica(replica.getBlockPoolId(), replica.getBlockId(), deleteSavedCopies);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
index 48ddcc2..f9e30e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
public class FsDatasetTestUtil {
public static File getFile(FsDatasetSpi<?> fsd, String bpid, long bid) {
- return ((FsDatasetImpl)fsd).getFile(bpid, bid);
+ return ((FsDatasetImpl)fsd).getFile(bpid, bid, false);
}
public static File getBlockFile(FsDatasetSpi<?> fsd, String bpid, Block b
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9eebaa0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 777779f..95404b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -71,7 +71,7 @@ public class TestLazyPersistFiles {
private static final int THREADPOOL_SIZE = 10;
private static final short REPL_FACTOR = 1;
- private static final int BLOCK_SIZE = 10485760; // 10 MB
+ private static final int BLOCK_SIZE = 5 * 1024 * 1024;
private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
private static final long HEARTBEAT_INTERVAL_SEC = 1;
private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
@@ -449,34 +449,51 @@ public class TestLazyPersistFiles {
* @throws InterruptedException
*/
@Test (timeout=300000)
- public void testRamDiskEvictionLRU()
+ public void testRamDiskEvictionIsLru()
throws IOException, InterruptedException {
- startUpCluster(true, 3);
+ final int NUM_PATHS = 5;
+ startUpCluster(true, NUM_PATHS + EVICTION_LOW_WATERMARK);
final String METHOD_NAME = GenericTestUtils.getMethodName();
- final int NUM_PATHS = 6;
- Path paths[] = new Path[NUM_PATHS];
+ Path paths[] = new Path[NUM_PATHS * 2];
- for (int i = 0; i < NUM_PATHS; i++) {
+ for (int i = 0; i < paths.length; i++) {
paths[i] = new Path("/" + METHOD_NAME + "." + i +".dat");
}
- // No eviction for the first half of files
- for (int i = 0; i < NUM_PATHS/2; i++) {
+ for (int i = 0; i < NUM_PATHS; i++) {
makeTestFile(paths[i], BLOCK_SIZE, true);
- ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
}
- // Lazy persist writer persists the first half of files
+ // Sleep for a short time to allow the lazy writer thread to do its job.
Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
- // Create the second half of files with eviction upon each create.
- for (int i = NUM_PATHS/2; i < NUM_PATHS; i++) {
- makeTestFile(paths[i], BLOCK_SIZE, true);
+ for (int i = 0; i < NUM_PATHS; ++i) {
ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
+ }
+
+ // Open the files for read in a random order.
+ ArrayList<Integer> indexes = new ArrayList<Integer>(NUM_PATHS);
+ for (int i = 0; i < NUM_PATHS; ++i) {
+ indexes.add(i);
+ }
+ Collections.shuffle(indexes);
- // path[i-NUM_PATHS/2] is expected to be evicted by LRU
+ for (int i = 0; i < NUM_PATHS; ++i) {
+ LOG.info("Touching file " + paths[indexes.get(i)]);
+ DFSTestUtil.readFile(fs, paths[indexes.get(i)]);
+ }
+
+ // Create an equal number of new files ensuring that the previous
+ // files are evicted in the same order they were read.
+ for (int i = 0; i < NUM_PATHS; ++i) {
+ makeTestFile(paths[i + NUM_PATHS], BLOCK_SIZE, true);
triggerBlockReport();
- ensureFileReplicasOnStorageType(paths[i - NUM_PATHS / 2], DEFAULT);
+ Thread.sleep(3000);
+ ensureFileReplicasOnStorageType(paths[i + NUM_PATHS], RAM_DISK);
+ ensureFileReplicasOnStorageType(paths[indexes.get(i)], DEFAULT);
+ for (int j = i + 1; j < NUM_PATHS; ++j) {
+ ensureFileReplicasOnStorageType(paths[indexes.get(j)], RAM_DISK);
+ }
}
}
[07/34] git commit: HDFS-6928. 'hdfs put' command should accept
lazyPersist flag for testing. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2079d4b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2079d4b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2079d4b7
Branch: refs/heads/branch-2.6
Commit: 2079d4b72e1a61bad227384bd7ab778f139d6443
Parents: 15372df
Author: arp <ar...@apache.org>
Authored: Thu Aug 28 15:53:12 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/FilterFileSystem.java | 13 ++++++-
.../hadoop/fs/shell/CommandWithDestination.java | 32 ++++++++++++++---
.../apache/hadoop/fs/shell/CopyCommands.java | 16 +++++----
.../java/org/apache/hadoop/fs/shell/Stat.java | 7 +++-
.../src/test/resources/testConf.xml | 28 +++++++++++----
.../src/test/resources/testHDFSConf.xml | 36 ++++++++++++++++++++
6 files changed, 113 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 52706f4..e729e67 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -179,8 +179,19 @@ public class FilterFileSystem extends FileSystem {
return fs.create(f, permission,
overwrite, bufferSize, replication, blockSize, progress);
}
-
+ @Override
+ public FSDataOutputStream create(Path f,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
+ return fs.create(f, permission,
+ flags, bufferSize, replication, blockSize, progress);
+ }
@Override
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index da67f1c..5d0d9d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
@@ -45,6 +46,9 @@ import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+
/**
* Provides: argument processing to ensure the destination is valid
* for the number of source arguments. A processPaths that accepts both
@@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand {
private boolean overwrite = false;
private boolean verifyChecksum = true;
private boolean writeChecksum = true;
+ private boolean lazyPersist = false;
/**
* The name of the raw xattr namespace. It would be nice to use
@@ -78,6 +83,10 @@ abstract class CommandWithDestination extends FsCommand {
overwrite = flag;
}
+ protected void setLazyPersist(boolean flag) {
+ lazyPersist = flag;
+ }
+
protected void setVerifyChecksum(boolean flag) {
verifyChecksum = flag;
}
@@ -379,7 +388,7 @@ abstract class CommandWithDestination extends FsCommand {
try {
PathData tempTarget = target.suffix("._COPYING_");
targetFs.setWriteChecksum(writeChecksum);
- targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.writeStreamToFile(in, tempTarget, lazyPersist);
targetFs.rename(tempTarget, target);
} finally {
targetFs.close(); // last ditch effort to ensure temp file is removed
@@ -449,10 +458,11 @@ abstract class CommandWithDestination extends FsCommand {
super(fs);
}
- void writeStreamToFile(InputStream in, PathData target) throws IOException {
+ void writeStreamToFile(InputStream in, PathData target,
+ boolean lazyPersist) throws IOException {
FSDataOutputStream out = null;
try {
- out = create(target);
+ out = create(target, lazyPersist);
IOUtils.copyBytes(in, out, getConf(), true);
} finally {
IOUtils.closeStream(out); // just in case copyBytes didn't
@@ -460,9 +470,21 @@ abstract class CommandWithDestination extends FsCommand {
}
// tag created files as temp files
- FSDataOutputStream create(PathData item) throws IOException {
+ FSDataOutputStream create(PathData item, boolean lazyPersist)
+ throws IOException {
try {
- return create(item.path, true);
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+ if (lazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+ return create(item.path,
+ null,
+ createFlags,
+ getConf().getInt("io.file.buffer.size", 4096),
+ lazyPersist ? 1 : getDefaultReplication(item.path),
+ getDefaultBlockSize(),
+ null,
+ null);
} finally { // might have been created but stream was interrupted
deleteOnExit(item.path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 3fd870c..afd1115 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -215,21 +215,25 @@ class CopyCommands {
*/
public static class Put extends CommandWithDestination {
public static final String NAME = "put";
- public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
+ public static final String USAGE = "[-f] [-p] [-l] <localsrc> ... <dst>";
public static final String DESCRIPTION =
"Copy files from the local file system " +
"into fs. Copying fails if the file already " +
- "exists, unless the -f flag is given. Passing " +
- "-p preserves access and modification times, " +
- "ownership and the mode. Passing -f overwrites " +
- "the destination if it already exists.\n";
+ "exists, unless the -f flag is given.\n" +
+ "Flags:\n" +
+ " -p : Preserves access and modification times, ownership and the mode.\n" +
+ " -f : Overwrites the destination if it already exists.\n" +
+ " -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
+ " replication factor of 1. This flag will result in reduced\n" +
+ " durability. Use with care.\n";
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
+ CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l");
cf.parse(args);
setOverwrite(cf.getOpt("f"));
setPreserve(cf.getOpt("p"));
+ setLazyPersist(cf.getOpt("l"));
getRemoteDestination(args);
// should have a -r option
setRecursive(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index 652c928..8b32eb8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
* %u: User name of owner
* %y: UTC date as "yyyy-MM-dd HH:mm:ss"
* %Y: Milliseconds since January 1, 1970 UTC
+ * %l: Whether lazyPersist flag is set on the file.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -53,7 +54,8 @@ class Stat extends FsCommand {
public static final String DESCRIPTION =
"Print statistics about the file/directory at <path> " +
"in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
- "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
+ "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " +
+ "lazyPersist flag (%l)\n";
protected static final SimpleDateFormat timeFmt;
static {
@@ -115,6 +117,9 @@ class Stat extends FsCommand {
case 'Y':
buf.append(stat.getModificationTime());
break;
+ case 'l':
+ buf.append(stat.isLazyPersist());
+ break;
default:
// this leaves %<unknown> alone, which causes the potential for
// future format options to break strings; should use %% to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 804b504..29d4174 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -436,7 +436,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-put \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s*</expected-output>
+ <expected-output>^-put \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -444,15 +444,31 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )*</expected-output>
+ <expected-output>^\s*exists, unless the -f flag is given.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*modification times, ownership and the mode. Passing -f overwrites the( )*</expected-output>
+ <expected-output>^\s*Flags:( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*destination if it already exists.( )*</expected-output>
+ <expected-output>^\s*-p Preserves access and modification times, ownership and the mode.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*-f Overwrites the destination if it already exists.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*-l Allow DataNode to lazily persist the file to disk. Forces( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*replication factor of 1. This flag will result in reduced( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*durability. Use with care.( )*</expected-output>
</comparator>
</comparators>
</test>
@@ -467,7 +483,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-copyFromLocal \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s*</expected-output>
+ <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -782,7 +798,7 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
+ <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
</comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2079d4b7/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 2c2a7a0..37ea234 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -4554,6 +4554,42 @@
</comparators>
</test>
+ <test> <!-- TESTED -->
+ <description>put: The LazyPersist flag is set when requested</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dirLp</command>
+ <command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+ <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dirLp/</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^true</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>put: The LazyPersist flag is not set by default</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dirLp</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+ <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dirLp/</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^false</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
<!-- Tests for copyFromLocal -->
<test> <!-- TESTED -->
<description>copyFromLocal: copying file into a file (absolute path)</description>
[24/34] git commit: HDFS-7159. Use block storage policy to set lazy
persist preference. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7159. Use block storage policy to set lazy persist preference. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d61daff9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d61daff9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d61daff9
Branch: refs/heads/branch-2.6
Commit: d61daff9bf9264ed204829352bb570a7dc7e8fe0
Parents: ba31a95
Author: arp <ar...@apache.org>
Authored: Mon Sep 29 22:27:59 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/Stat.java | 9 +--
.../src/test/resources/testConf.xml | 2 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-
.../org/apache/hadoop/hdfs/StorageType.java | 8 ++-
.../hdfs/protocol/BlockStoragePolicy.java | 37 ++++++++++--
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +
.../server/blockmanagement/BlockCollection.java | 6 --
.../server/blockmanagement/BlockManager.java | 4 ++
.../BlockStoragePolicySuite.java | 7 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
.../fsdataset/impl/FsTransientVolumeImpl.java | 60 --------------------
.../datanode/fsdataset/impl/FsVolumeImpl.java | 6 +-
.../fsdataset/impl/FsVolumeImplAllocator.java | 44 --------------
.../hdfs/server/namenode/FSDirectory.java | 28 ++++++---
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 -
.../hdfs/server/namenode/FSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/FSEditLogOp.java | 23 +-------
.../hdfs/server/namenode/FSImageFormat.java | 6 +-
.../server/namenode/FSImageFormatPBINode.java | 5 +-
.../hdfs/server/namenode/FSNamesystem.java | 45 ++++++++++++++-
.../hadoop/hdfs/server/namenode/INodeFile.java | 10 ++--
.../server/namenode/INodeFileAttributes.java | 6 +-
.../server/namenode/NameNodeLayoutVersion.java | 2 -
.../snapshot/FSImageFormatPBSnapshot.java | 2 -
.../tools/offlineImageViewer/FSImageLoader.java | 1 -
.../offlineImageViewer/PBImageXmlWriter.java | 4 --
.../hadoop-hdfs/src/main/proto/fsimage.proto | 1 -
.../fsdataset/impl/TestLazyPersistFiles.java | 22 ++++---
.../fsdataset/impl/TestScrLazyPersistFiles.java | 17 +-----
.../src/test/resources/testHDFSConf.xml | 36 ------------
30 files changed, 160 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index 8b32eb8..ee56fe6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.FileStatus;
* %u: User name of owner
* %y: UTC date as "yyyy-MM-dd HH:mm:ss"
* %Y: Milliseconds since January 1, 1970 UTC
- * %l: Whether lazyPersist flag is set on the file.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -54,8 +53,7 @@ class Stat extends FsCommand {
public static final String DESCRIPTION =
"Print statistics about the file/directory at <path> " +
"in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
- "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " +
- "lazyPersist flag (%l)\n";
+ "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
protected static final SimpleDateFormat timeFmt;
static {
@@ -93,7 +91,7 @@ class Stat extends FsCommand {
break;
case 'F':
buf.append(stat.isDirectory()
- ? "directory"
+ ? "directory"
: (stat.isFile() ? "regular file" : "symlink"));
break;
case 'g':
@@ -117,9 +115,6 @@ class Stat extends FsCommand {
case 'Y':
buf.append(stat.getModificationTime());
break;
- case 'l':
- buf.append(stat.isLazyPersist());
- break;
default:
// this leaves %<unknown> alone, which causes the potential for
// future format options to break strings; should use %% to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 29d4174..c6e5fc5 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -798,7 +798,7 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
+ <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
</comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 2bcda8b..a37e4d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -75,6 +76,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
import org.apache.hadoop.hdfs.server.namenode.RetryStartFileException;
@@ -172,6 +174,8 @@ public class DFSOutputStream extends FSOutputSummer
private final AtomicReference<CachingStrategy> cachingStrategy;
private boolean failPacket = false;
private FileEncryptionInfo fileEncryptionInfo;
+ private static final BlockStoragePolicySuite blockStoragePolicySuite =
+ BlockStoragePolicySuite.createDefaultSuite();
private static class Packet {
private static final long HEART_BEAT_SEQNO = -1L;
@@ -386,7 +390,7 @@ public class DFSOutputStream extends FSOutputSummer
*/
private DataStreamer(HdfsFileStatus stat, Span span) {
isAppend = false;
- isLazyPersistFile = stat.isLazyPersist();
+ isLazyPersistFile = initLazyPersist(stat);
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
traceSpan = span;
}
@@ -406,7 +410,7 @@ public class DFSOutputStream extends FSOutputSummer
block = lastBlock.getBlock();
bytesSent = block.getNumBytes();
accessToken = lastBlock.getBlockToken();
- isLazyPersistFile = stat.isLazyPersist();
+ isLazyPersistFile = initLazyPersist(stat);
long usedInLastBlock = stat.getLen() % blockSize;
int freeInLastBlock = (int)(blockSize - usedInLastBlock);
@@ -450,6 +454,13 @@ public class DFSOutputStream extends FSOutputSummer
}
}
+ private boolean initLazyPersist(HdfsFileStatus stat) {
+ final BlockStoragePolicy lpPolicy =
+ blockStoragePolicySuite.getPolicy("LAZY_PERSIST");
+ return lpPolicy != null &&
+ stat.getStoragePolicy() == lpPolicy.getId();
+ }
+
private void setPipeline(LocatedBlock lb) {
setPipeline(lb.getLocations(), lb.getStorageTypes(), lb.getStorageIDs());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index 99cae9a..88cc7d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -47,7 +47,13 @@ public enum StorageType {
StorageType(boolean isTransient) { this.isTransient = isTransient; }
- public boolean isMovable() { return isTransient == false; }
+ public boolean isTransient() {
+ return isTransient;
+ }
+
+ public boolean isMovable() {
+ return !isTransient;
+ }
public static List<StorageType> asList() {
return Arrays.asList(VALUES);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
index 8ca83a0..ef13e0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
@@ -49,15 +49,29 @@ public class BlockStoragePolicy {
private final StorageType[] creationFallbacks;
/** The fallback storage type for replication. */
private final StorageType[] replicationFallbacks;
+ /**
+ * Whether the policy is inherited during file creation.
+ * If set then the policy cannot be changed after file creation.
+ */
+ private boolean copyOnCreateFile;
@VisibleForTesting
public BlockStoragePolicy(byte id, String name, StorageType[] storageTypes,
StorageType[] creationFallbacks, StorageType[] replicationFallbacks) {
+ this(id, name, storageTypes, creationFallbacks, replicationFallbacks,
+ false);
+ }
+
+ @VisibleForTesting
+ public BlockStoragePolicy(byte id, String name, StorageType[] storageTypes,
+ StorageType[] creationFallbacks, StorageType[] replicationFallbacks,
+ boolean copyOnCreateFile) {
this.id = id;
this.name = name;
this.storageTypes = storageTypes;
this.creationFallbacks = creationFallbacks;
this.replicationFallbacks = replicationFallbacks;
+ this.copyOnCreateFile = copyOnCreateFile;
}
/**
@@ -65,13 +79,22 @@ public class BlockStoragePolicy {
*/
public List<StorageType> chooseStorageTypes(final short replication) {
final List<StorageType> types = new LinkedList<StorageType>();
- int i = 0;
- for(; i < replication && i < storageTypes.length; i++) {
- types.add(storageTypes[i]);
+ int i = 0, j = 0;
+
+ // Do not return transient storage types. We will not have accurate
+ // usage information for transient types.
+ for (;i < replication && j < storageTypes.length; ++j) {
+ if (!storageTypes[j].isTransient()) {
+ types.add(storageTypes[j]);
+ ++i;
+ }
}
+
final StorageType last = storageTypes[storageTypes.length - 1];
- for(; i < replication; i++) {
- types.add(last);
+ if (!last.isTransient()) {
+ for (; i < replication; i++) {
+ types.add(last);
+ }
}
return types;
}
@@ -241,4 +264,8 @@ public class BlockStoragePolicy {
}
return null;
}
+
+ public boolean isCopyOnCreateFile() {
+ return copyOnCreateFile;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 25c64d3..b0d1dca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -171,7 +171,9 @@ public class HdfsConstants {
public static final String HOT_STORAGE_POLICY_NAME = "HOT";
public static final String WARM_STORAGE_POLICY_NAME = "WARM";
public static final String COLD_STORAGE_POLICY_NAME = "COLD";
+ public static final String LAZY_PERSIST_STORAGE_POLICY_NAME = "LAZY_PERSIST";
+ public static final byte LAZY_PERSIST_STORAGE_POLICY_ID = 15;
public static final byte ALLSSD_STORAGE_POLICY_ID = 12;
public static final byte ONESSD_STORAGE_POLICY_ID = 10;
public static final byte HOT_STORAGE_POLICY_ID = 7;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index c97dcb9..d095a1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -55,12 +55,6 @@ public interface BlockCollection {
public long getPreferredBlockSize();
/**
- * Return true if the file was created with {@Link CreateFlag#LAZY_PERSIST}.
- * @return
- */
- public boolean getLazyPersistFlag();
-
- /**
* Get block replication for the collection
* @return block replication value
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index e95f705..99710d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -402,6 +402,10 @@ public class BlockManager {
return storagePolicySuite.getPolicy(policyName);
}
+ public BlockStoragePolicy getStoragePolicy(final byte policyId) {
+ return storagePolicySuite.getPolicy(policyId);
+ }
+
public BlockStoragePolicy[] getStoragePolicies() {
return storagePolicySuite.getAllPolicies();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index cbd1823..fa94921 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -45,6 +45,13 @@ public class BlockStoragePolicySuite {
public static BlockStoragePolicySuite createDefaultSuite() {
final BlockStoragePolicy[] policies =
new BlockStoragePolicy[1 << ID_BIT_LENGTH];
+ final byte lazyPersistId = HdfsConstants.LAZY_PERSIST_STORAGE_POLICY_ID;
+ policies[lazyPersistId] = new BlockStoragePolicy(lazyPersistId,
+ HdfsConstants.LAZY_PERSIST_STORAGE_POLICY_NAME,
+ new StorageType[]{StorageType.RAM_DISK, StorageType.DISK},
+ new StorageType[]{StorageType.DISK},
+ new StorageType[]{StorageType.DISK},
+ true); // Cannot be changed on regular files, but inherited.
final byte allssdId = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
policies[allssdId] = new BlockStoragePolicy(allssdId,
HdfsConstants.ALLSSD_STORAGE_POLICY_NAME,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 11322a9..8ed81df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -295,7 +295,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// If IOException raises from FsVolumeImpl() or getVolumeMap(), there is
// nothing needed to be rolled back to make various data structures, e.g.,
// storageMap and asyncDiskService, consistent.
- FsVolumeImpl fsVolume = FsVolumeImplAllocator.createVolume(
+ FsVolumeImpl fsVolume = new FsVolumeImpl(
this, sd.getStorageUuid(), dir, this.conf, storageType);
ReplicaMap tempVolumeMap = new ReplicaMap(this);
fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
deleted file mode 100644
index dafa74f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.StorageType;
-
-/**
- * Volume for storing replicas in memory. These can be deleted at any time
- * to make space for new replicas and there is no persistence guarantee.
- *
- * The backing store for these replicas is expected to be RAM_DISK.
- * The backing store may be disk when testing.
- *
- * It uses the {@link FsDatasetImpl} object for synchronization.
- */
-@InterfaceAudience.Private
-@VisibleForTesting
-public class FsTransientVolumeImpl extends FsVolumeImpl {
-
-
- FsTransientVolumeImpl(FsDatasetImpl dataset, String storageID, File currentDir,
- Configuration conf, StorageType storageType)
- throws IOException {
- super(dataset, storageID, currentDir, conf, storageType);
- }
-
- @Override
- protected ThreadPoolExecutor initializeCacheExecutor(File parent) {
- // Can't 'cache' replicas already in RAM.
- return null;
- }
-
- @Override
- public boolean isTransientStorage() {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 098ad09..60ea125 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -96,6 +96,10 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
protected ThreadPoolExecutor initializeCacheExecutor(File parent) {
+ if (storageType.isTransient()) {
+ return null;
+ }
+
final int maxNumThreads = dataset.datanode.getConf().getInt(
DFSConfigKeys.DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY,
DFSConfigKeys.DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT);
@@ -203,7 +207,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
@Override
public boolean isTransientStorage() {
- return false;
+ return storageType.isTransient();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
deleted file mode 100644
index 14d3aaf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.StorageType;
-
-/**
- * Generate volumes based on the storageType.
- */
-@InterfaceAudience.Private
-class FsVolumeImplAllocator {
- static FsVolumeImpl createVolume(FsDatasetImpl fsDataset, String storageUuid,
- File dir, Configuration conf, StorageType storageType)
- throws IOException {
- switch(storageType) {
- case RAM_DISK:
- return new FsTransientVolumeImpl(
- fsDataset, storageUuid, dir, conf, storageType);
- default:
- return new FsVolumeImpl(
- fsDataset, storageUuid, dir, conf, storageType);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 56f9045..3e6cf5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
@@ -299,14 +300,13 @@ public class FSDirectory implements Closeable {
*/
INodeFile addFile(String path, PermissionStatus permissions,
short replication, long preferredBlockSize,
- boolean isLazyPersist,
String clientName, String clientMachine)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException, AclException {
long modTime = now();
INodeFile newNode = newINodeFile(namesystem.allocateNewInodeId(),
- permissions, modTime, modTime, replication, preferredBlockSize, isLazyPersist);
+ permissions, modTime, modTime, replication, preferredBlockSize);
newNode.toUnderConstruction(clientName, clientMachine);
boolean added = false;
@@ -336,7 +336,6 @@ public class FSDirectory implements Closeable {
long modificationTime,
long atime,
long preferredBlockSize,
- boolean isLazyPersist,
boolean underConstruction,
String clientName,
String clientMachine,
@@ -345,12 +344,12 @@ public class FSDirectory implements Closeable {
assert hasWriteLock();
if (underConstruction) {
newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize, storagePolicyId, isLazyPersist);
+ modificationTime, replication, preferredBlockSize, storagePolicyId);
newNode.toUnderConstruction(clientName, clientMachine);
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize, storagePolicyId, isLazyPersist);
+ replication, preferredBlockSize, storagePolicyId);
}
try {
@@ -1038,6 +1037,20 @@ public class FSDirectory implements Closeable {
}
final int snapshotId = iip.getLatestSnapshotId();
if (inode.isFile()) {
+ BlockStoragePolicy newPolicy = getBlockManager().getStoragePolicy(policyId);
+ if (newPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException(
+ "Policy " + newPolicy + " cannot be set after file creation.");
+ }
+
+ BlockStoragePolicy currentPolicy =
+ getBlockManager().getStoragePolicy(inode.getLocalStoragePolicyID());
+
+ if (currentPolicy != null && currentPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException(
+ "Existing policy " + currentPolicy.getName() +
+ " cannot be changed after file creation.");
+ }
inode.asFile().setStoragePolicyID(policyId, snapshotId);
} else if (inode.isDirectory()) {
setDirStoragePolicy(inode.asDirectory(), policyId, snapshotId);
@@ -2431,7 +2444,6 @@ public class FSDirectory implements Closeable {
node.isDirectory(),
replication,
blocksize,
- isLazyPersist,
node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot, isEncrypted),
@@ -2455,7 +2467,6 @@ public class FSDirectory implements Closeable {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
- boolean isLazyPersist = false;
LocatedBlocks loc = null;
final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
@@ -2465,7 +2476,6 @@ public class FSDirectory implements Closeable {
size = fileNode.computeFileSize(snapshot);
replication = fileNode.getFileReplication(snapshot);
blocksize = fileNode.getPreferredBlockSize();
- isLazyPersist = fileNode.getLazyPersistFlag();
final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID;
final boolean isUc = !inSnapshot && fileNode.isUnderConstruction();
@@ -2488,7 +2498,7 @@ public class FSDirectory implements Closeable {
HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
- blocksize, isLazyPersist, node.getModificationTime(snapshot),
+ blocksize, node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot, isEncrypted),
node.getUserName(snapshot), node.getGroupName(snapshot),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 1a57179..cb44011 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -714,7 +714,6 @@ public class FSEditLog implements LogsPurgeable {
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
- .setLazyPersistFlag(newNode.getLazyPersistFlag())
.setBlocks(newNode.getBlocks())
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
@@ -747,7 +746,6 @@ public class FSEditLog implements LogsPurgeable {
.setModificationTime(newNode.getModificationTime())
.setAccessTime(newNode.getAccessTime())
.setBlockSize(newNode.getPreferredBlockSize())
- .setLazyPersistFlag(newNode.getLazyPersistFlag())
.setBlocks(newNode.getBlocks())
.setPermissionStatus(newNode.getPermissionStatus());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 472a7f3..7dfe688 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -366,7 +366,7 @@ public class FSEditLogLoader {
path, addCloseOp.permissions, addCloseOp.aclEntries,
addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
- addCloseOp.blockSize, addCloseOp.isLazyPersist, true, addCloseOp.clientName,
+ addCloseOp.blockSize, true, addCloseOp.clientName,
addCloseOp.clientMachine, addCloseOp.storagePolicyId);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index f66530d..ec2b633 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -406,7 +406,6 @@ public abstract class FSEditLogOp {
long mtime;
long atime;
long blockSize;
- boolean isLazyPersist;
Block[] blocks;
PermissionStatus permissions;
List<AclEntry> aclEntries;
@@ -418,6 +417,7 @@ public abstract class FSEditLogOp {
private AddCloseOp(FSEditLogOpCodes opCode) {
super(opCode);
+ storagePolicyId = BlockStoragePolicySuite.ID_UNSPECIFIED;
assert(opCode == OP_ADD || opCode == OP_CLOSE);
}
@@ -456,11 +456,6 @@ public abstract class FSEditLogOp {
return (T)this;
}
- <T extends AddCloseOp> T setLazyPersistFlag(boolean isLazyPersist) {
- this.isLazyPersist = isLazyPersist;
- return (T)this;
- }
-
<T extends AddCloseOp> T setBlocks(Block[] blocks) {
if (blocks.length > MAX_BLOCKS) {
throw new RuntimeException("Can't have more than " + MAX_BLOCKS +
@@ -518,7 +513,6 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeLong(mtime, out);
FSImageSerialization.writeLong(atime, out);
FSImageSerialization.writeLong(blockSize, out);
- FSImageSerialization.writeInt((isLazyPersist ? 1 : 0), out);
new ArrayWritable(Block.class, blocks).write(out);
permissions.write(out);
@@ -588,13 +582,6 @@ public abstract class FSEditLogOp {
this.blockSize = readLong(in);
}
- if (NameNodeLayoutVersion.supports(
- NameNodeLayoutVersion.Feature.LAZY_PERSIST_FILES, logVersion)) {
- this.isLazyPersist = (FSImageSerialization.readInt(in) != 0);
- } else {
- this.isLazyPersist = false;
- }
-
this.blocks = readBlocks(in, logVersion);
this.permissions = PermissionStatus.read(in);
@@ -660,8 +647,6 @@ public abstract class FSEditLogOp {
builder.append(atime);
builder.append(", blockSize=");
builder.append(blockSize);
- builder.append(", lazyPersist");
- builder.append(isLazyPersist);
builder.append(", blocks=");
builder.append(Arrays.toString(blocks));
builder.append(", permissions=");
@@ -702,8 +687,6 @@ public abstract class FSEditLogOp {
Long.toString(atime));
XMLUtils.addSaxString(contentHandler, "BLOCKSIZE",
Long.toString(blockSize));
- XMLUtils.addSaxString(contentHandler, "LAZY_PERSIST",
- Boolean.toString(isLazyPersist));
XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName);
XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine);
XMLUtils.addSaxString(contentHandler, "OVERWRITE",
@@ -730,10 +713,6 @@ public abstract class FSEditLogOp {
this.atime = Long.parseLong(st.getValue("ATIME"));
this.blockSize = Long.parseLong(st.getValue("BLOCKSIZE"));
- String lazyPersistString = st.getValueOrNull("LAZY_PERSIST");
- this.isLazyPersist =
- lazyPersistString != null && Boolean.parseBoolean(lazyPersistString);
-
this.clientName = st.getValue("CLIENT_NAME");
this.clientMachine = st.getValue("CLIENT_MACHINE");
this.overwrite = Boolean.parseBoolean(st.getValueOrNull("OVERWRITE"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index fd0f467..3c58ed9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -786,8 +786,6 @@ public class FSImageFormat {
counter.increment();
}
- // Images in the old format will not have the lazyPersist flag so it is
- // safe to pass false always.
final INodeFile file = new INodeFile(inodeId, localName, permissions,
modificationTime, atime, blocks, replication, blockSize, (byte)0);
if (underConstruction) {
@@ -892,10 +890,8 @@ public class FSImageFormat {
in.readShort());
final long preferredBlockSize = in.readLong();
- // LazyPersist flag will not be present in old image formats and hence
- // can be safely set to false always.
return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
- accessTime, replication, preferredBlockSize, (byte)0, false null);
+ accessTime, replication, preferredBlockSize, (byte) 0, null);
}
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 8cce6d2..5631b2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -290,7 +290,7 @@ public final class FSImageFormatPBINode {
final INodeFile file = new INodeFile(n.getId(),
n.getName().toByteArray(), permissions, f.getModificationTime(),
f.getAccessTime(), blocks, replication, f.getPreferredBlockSize(),
- (byte)f.getStoragePolicyID(), f.hasIsLazyPersist() ? f.getIsLazyPersist() : false);
+ (byte)f.getStoragePolicyID());
if (f.hasAcl()) {
file.addAclFeature(new AclFeature(loadAclEntries(f.getAcl(),
@@ -400,8 +400,7 @@ public final class FSImageFormatPBINode {
.setPermission(buildPermissionStatus(file, state.getStringMap()))
.setPreferredBlockSize(file.getPreferredBlockSize())
.setReplication(file.getFileReplication())
- .setStoragePolicyID(file.getLocalStoragePolicyID())
- .setIsLazyPersist(file.getLazyPersistFlag());
+ .setStoragePolicyID(file.getLocalStoragePolicyID());
AclFeature f = file.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7dd3874..3639b2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2356,6 +2356,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
src = FSDirectory.resolvePath(src, pathComponents, dir);
+ INode inode = dir.getINode(src);
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = blockManager.getStoragePolicy(policyName);
@@ -2741,7 +2742,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if (parent != null && mkdirsRecursively(parent.toString(),
permissions, true, now())) {
newNode = dir.addFile(src, permissions, replication, blockSize,
- isLazyPersist, holder, clientMachine);
+ holder, clientMachine);
}
if (newNode == null) {
@@ -2756,6 +2757,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
newNode = dir.getInode(newNode.getId()).asFile();
}
+ setNewINodeStoragePolicy(newNode, iip, isLazyPersist);
+
// record file record in log, record new generation stamp
getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry);
if (NameNode.stateChangeLog.isDebugEnabled()) {
@@ -2770,6 +2773,37 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
}
+ private void setNewINodeStoragePolicy(INodeFile inode,
+ INodesInPath iip,
+ boolean isLazyPersist)
+ throws IOException {
+
+ if (isLazyPersist) {
+ BlockStoragePolicy lpPolicy =
+ blockManager.getStoragePolicy("LAZY_PERSIST");
+
+ // Set LAZY_PERSIST storage policy if the flag was passed to
+ // CreateFile.
+ if (lpPolicy == null) {
+ throw new HadoopIllegalArgumentException(
+ "The LAZY_PERSIST storage policy has been disabled " +
+ "by the administrator.");
+ }
+ inode.setStoragePolicyID(lpPolicy.getId(),
+ iip.getLatestSnapshotId());
+ } else {
+ BlockStoragePolicy effectivePolicy =
+ blockManager.getStoragePolicy(inode.getStoragePolicyID());
+
+ if (effectivePolicy != null &&
+ effectivePolicy.isCopyOnCreateFile()) {
+ // Copy effective policy from ancestor directory to current file.
+ inode.setStoragePolicyID(effectivePolicy.getId(),
+ iip.getLatestSnapshotId());
+ }
+ }
+ }
+
/**
* Append to an existing file for append.
* <p>
@@ -2809,8 +2843,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
+ src + " for client " + clientMachine);
}
INodeFile myFile = INodeFile.valueOf(inode, src, true);
+ final BlockStoragePolicy lpPolicy =
+ blockManager.getStoragePolicy("LAZY_PERSIST");
- if (myFile.getLazyPersistFlag()) {
+ if (lpPolicy != null &&
+ lpPolicy.getId() == myFile.getStoragePolicyID()) {
throw new UnsupportedOperationException(
"Cannot append to lazy persist file " + src);
}
@@ -5166,6 +5203,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
throws SafeModeException, AccessControlException,
UnresolvedLinkException, IOException {
+ BlockStoragePolicy lpPolicy = blockManager.getStoragePolicy("LAZY_PERSIST");
+
List<BlockCollection> filesToDelete = new ArrayList<BlockCollection>();
writeLock();
@@ -5176,7 +5215,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
while (it.hasNext()) {
Block b = it.next();
BlockInfo blockInfo = blockManager.getStoredBlock(b);
- if (blockInfo.getBlockCollection().getLazyPersistFlag()) {
+ if (blockInfo.getBlockCollection().getStoragePolicyID() == lpPolicy.getId()) {
filesToDelete.add(blockInfo.getBlockCollection());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index aeddcad..b04c0bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -80,8 +80,7 @@ public class INodeFile extends INodeWithAdditionalFields
PREFERRED_BLOCK_SIZE(null, 48, 1),
REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 12, 1),
STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH,
- 0),
- LAZY_PERSIST(REPLICATION.BITS, 4, 0);
+ 0);
private final LongBitFormat BITS;
@@ -107,7 +106,6 @@ public class INodeFile extends INodeWithAdditionalFields
h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
h = REPLICATION.BITS.combine(replication, h);
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
- h = LAZY_PERSIST.BITS.combine(isLazyPersist ? 1 : 0, h);
return h;
}
static boolean getLazyPersistFlag(long header) {
@@ -123,15 +121,15 @@ public class INodeFile extends INodeWithAdditionalFields
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
this(id, name, permissions, mtime, atime, blklist, replication,
- preferredBlockSize, false);
+ preferredBlockSize, (byte) 0);
}
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
long atime, BlockInfo[] blklist, short replication,
- long preferredBlockSize, byte storagePolicyID, boolean isLazyPersist) {
+ long preferredBlockSize, byte storagePolicyID) {
super(id, name, permissions, mtime, atime);
header = HeaderFormat.toLong(preferredBlockSize, replication,
- storagePolicyID, isLazyPersist);
+ storagePolicyID);
this.blocks = blklist;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
index 05ce78f..9577d1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
@@ -32,8 +32,6 @@ public interface INodeFileAttributes extends INodeAttributes {
/** @return preferred block size in bytes */
public long getPreferredBlockSize();
- public boolean getLazyPersistFlag();
-
/** @return the header as a long. */
public long getHeaderLong();
@@ -49,10 +47,10 @@ public interface INodeFileAttributes extends INodeAttributes {
public SnapshotCopy(byte[] name, PermissionStatus permissions,
AclFeature aclFeature, long modificationTime, long accessTime,
short replication, long preferredBlockSize, byte storagePolicyID,
- boolean isTransient, XAttrFeature xAttrsFeature) {
+ XAttrFeature xAttrsFeature) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
- header = HeaderFormat.toLong(preferredBlockSize, replication, isTransient, storagePolicyID);
+ header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
}
public SnapshotCopy(INodeFile file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 72fc9df..512913b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -70,8 +70,6 @@ public class NameNodeLayoutVersion {
"creating file with overwrite"),
XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"),
BLOCK_STORAGE_POLICY(-60, "Block Storage policy");
- LAZY_PERSIST_FILES(-61, "Support for optional lazy persistence of "
- + " files with reduced durability guarantees");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index b1727c6..e7598d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -222,9 +222,7 @@ public class FSImageFormatPBSnapshot {
fileInPb.getAccessTime(), (short) fileInPb.getReplication(),
fileInPb.getPreferredBlockSize(),
(byte)fileInPb.getStoragePolicyID(),
- fileInPb.hasIsLazyPersist() ? fileInPb.getIsLazyPersist() : false,
xAttrs);
-
}
FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 9297c64..b68d842 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -434,7 +434,6 @@ class FSImageLoader {
f.getPermission(), stringTable);
map.put("accessTime", f.getAccessTime());
map.put("blockSize", f.getPreferredBlockSize());
- map.put("lazyPersist", f.getIsLazyPersist());
map.put("group", p.getGroupName());
map.put("length", getFileSize(f));
map.put("modificationTime", f.getModificationTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 744fc75..99617b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -247,10 +247,6 @@ public final class PBImageXmlWriter {
.o("perferredBlockSize", f.getPreferredBlockSize())
.o("permission", dumpPermission(f.getPermission()));
- if (f.hasIsLazyPersist()) {
- o("lazyPersist", f.getIsLazyPersist());
- }
-
if (f.getBlocksCount() > 0) {
out.print("<blocks>");
for (BlockProto b : f.getBlocksList()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 85edfab..588f6c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -139,7 +139,6 @@ message INodeSection {
optional AclFeatureProto acl = 8;
optional XAttrFeatureProto xAttrs = 9;
optional uint32 storagePolicyID = 10;
- optional bool isLazyPersist = 11 [default = false];
}
message INodeDirectory {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 95404b3..928d0d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -56,6 +56,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -68,6 +69,8 @@ public class TestLazyPersistFiles {
((Log4JLogger) FsDatasetImpl.LOG).getLogger().setLevel(Level.ALL);
}
+ private static final byte LAZY_PERSIST_POLICY_ID = (byte) 15;
+
private static final int THREADPOOL_SIZE = 10;
private static final short REPL_FACTOR = 1;
@@ -100,19 +103,20 @@ public class TestLazyPersistFiles {
}
@Test (timeout=300000)
- public void testFlagNotSetByDefault() throws IOException {
+ public void testPolicyNotSetByDefault() throws IOException {
startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
makeTestFile(path, 0, false);
- // Stat the file and check that the lazyPersist flag is returned back.
+ // Stat the file and check that the LAZY_PERSIST policy is not
+ // returned back.
HdfsFileStatus status = client.getFileInfo(path.toString());
- assertThat(status.isLazyPersist(), is(false));
+ assertThat(status.getStoragePolicy(), not(LAZY_PERSIST_POLICY_ID));
}
@Test (timeout=300000)
- public void testFlagPropagation() throws IOException {
+ public void testPolicyPropagation() throws IOException {
startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -120,11 +124,11 @@ public class TestLazyPersistFiles {
makeTestFile(path, 0, true);
// Stat the file and check that the lazyPersist flag is returned back.
HdfsFileStatus status = client.getFileInfo(path.toString());
- assertThat(status.isLazyPersist(), is(true));
+ assertThat(status.getStoragePolicy(), is(LAZY_PERSIST_POLICY_ID));
}
@Test (timeout=300000)
- public void testFlagPersistenceInEditLog() throws IOException {
+ public void testPolicyPersistenceInEditLog() throws IOException {
startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -134,11 +138,11 @@ public class TestLazyPersistFiles {
// Stat the file and check that the lazyPersist flag is returned back.
HdfsFileStatus status = client.getFileInfo(path.toString());
- assertThat(status.isLazyPersist(), is(true));
+ assertThat(status.getStoragePolicy(), is(LAZY_PERSIST_POLICY_ID));
}
@Test (timeout=300000)
- public void testFlagPersistenceInFsImage() throws IOException {
+ public void testPolicyPersistenceInFsImage() throws IOException {
startUpCluster(false, -1);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -152,7 +156,7 @@ public class TestLazyPersistFiles {
// Stat the file and check that the lazyPersist flag is returned back.
HdfsFileStatus status = client.getFileInfo(path.toString());
- assertThat(status.isLazyPersist(), is(true));
+ assertThat(status.getStoragePolicy(), is(LAZY_PERSIST_POLICY_ID));
}
@Test (timeout=300000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
index 4270f8c..b6ac287 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
@@ -255,7 +255,7 @@ public class TestScrLazyPersistFiles {
for (FsVolumeSpi volume : volumes) {
if (volume.getStorageType() == RAM_DISK) {
- ((FsTransientVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
+ ((FsVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
}
}
}
@@ -263,13 +263,6 @@ public class TestScrLazyPersistFiles {
LOG.info("Cluster startup complete");
}
- private void startUpCluster(final int numDataNodes,
- final StorageType[] storageTypes,
- final long ramDiskStorageLimit)
- throws IOException {
- startUpCluster(numDataNodes, storageTypes, ramDiskStorageLimit, false);
- }
-
private void makeTestFile(Path path, long length, final boolean isLazyPersist)
throws IOException {
@@ -324,14 +317,6 @@ public class TestScrLazyPersistFiles {
BLOCK_SIZE, REPL_FACTOR, seed, true);
}
- private boolean verifyReadRandomFile(
- Path path, int fileLength, int seed) throws IOException {
- byte contents[] = DFSTestUtil.readFileBuffer(fs, path);
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(seed, fileLength);
- return Arrays.equals(contents, expected);
- }
-
private void triggerBlockReport()
throws IOException, InterruptedException {
// Trigger block report to NN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d61daff9/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 37ea234..2c2a7a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -4554,42 +4554,6 @@
</comparators>
</test>
- <test> <!-- TESTED -->
- <description>put: The LazyPersist flag is set when requested</description>
- <test-commands>
- <command>-fs NAMENODE -mkdir /dirLp</command>
- <command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
- <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
- </test-commands>
- <cleanup-commands>
- <command>-fs NAMENODE -rm -r /dirLp/</command>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^true</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>put: The LazyPersist flag is not set by default</description>
- <test-commands>
- <command>-fs NAMENODE -mkdir /dirLp</command>
- <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
- <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
- </test-commands>
- <cleanup-commands>
- <command>-fs NAMENODE -rm -r /dirLp/</command>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^false</expected-output>
- </comparator>
- </comparators>
- </test>
-
<!-- Tests for copyFromLocal -->
<test> <!-- TESTED -->
<description>copyFromLocal: copying file into a file (absolute path)</description>
[04/34] git commit: HDFS-6927. Initial unit tests for Lazy Persist
files. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6927. Initial unit tests for Lazy Persist files. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2721edc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2721edc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2721edc
Branch: refs/heads/branch-2.6
Commit: c2721edc50fdbb15e0882b8905e0851960000da1
Parents: 8e01a89
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:38:11 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../fsdataset/impl/TestLazyPersistFiles.java | 396 +++++++++++++++++++
1 file changed, 396 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2721edc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
new file mode 100644
index 0000000..6e7c091
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
+import static org.apache.hadoop.hdfs.StorageType.RAM_DISK;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Test;
+
+public class TestLazyPersistFiles {
+ public static final Log LOG = LogFactory.getLog(TestLazyPersistFiles.class);
+
+ static {
+ ((Log4JLogger) NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+ }
+
+ private static short REPL_FACTOR = 1;
+ private static final long BLOCK_SIZE = 10485760; // 10 MB
+ private static final int LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC = 3;
+ private static final long HEARTBEAT_INTERVAL_SEC = 1;
+ private static final int HEARTBEAT_RECHECK_INTERVAL_MSEC = 500;
+ private static final int LAZY_WRITER_INTERVAL_SEC = 3;
+ private static final int BUFFER_LENGTH = 4096;
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private DFSClient client;
+ private Configuration conf;
+
+ @After
+ public void shutDownCluster() throws IOException {
+ if (fs != null) {
+ fs.close();
+ fs = null;
+ client = null;
+ }
+
+ if (cluster != null) {
+ cluster.shutdownDataNodes();
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test (timeout=300000)
+ public void testFlagNotSetByDefault() throws IOException {
+ startUpCluster(REPL_FACTOR, null, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, 0, false);
+ // Stat the file and check that the lazyPersist flag is returned back.
+ HdfsFileStatus status = client.getFileInfo(path.toString());
+ assertThat(status.isLazyPersist(), is(false));
+ }
+
+ @Test (timeout=300000)
+ public void testFlagPropagation() throws IOException {
+ startUpCluster(REPL_FACTOR, null, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, 0, true);
+ // Stat the file and check that the lazyPersist flag is returned back.
+ HdfsFileStatus status = client.getFileInfo(path.toString());
+ assertThat(status.isLazyPersist(), is(true));
+ }
+
+ @Test (timeout=300000)
+ public void testFlagPersistenceInEditLog() throws IOException {
+ startUpCluster(REPL_FACTOR, null, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, 0, true);
+ cluster.restartNameNode(true);
+
+ // Stat the file and check that the lazyPersist flag is returned back.
+ HdfsFileStatus status = client.getFileInfo(path.toString());
+ assertThat(status.isLazyPersist(), is(true));
+ }
+
+ @Test (timeout=300000)
+ public void testFlagPersistenceInFsImage() throws IOException {
+ startUpCluster(REPL_FACTOR, null, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ FSDataOutputStream fos = null;
+
+ makeTestFile(path, 0, true);
+ // checkpoint
+ fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.restartNameNode(true);
+
+ // Stat the file and check that the lazyPersist flag is returned back.
+ HdfsFileStatus status = client.getFileInfo(path.toString());
+ assertThat(status.isLazyPersist(), is(true));
+ }
+
+ @Test (timeout=300000)
+ public void testPlacementOnRamDisk() throws IOException {
+ startUpCluster(REPL_FACTOR, new StorageType[] { DEFAULT, RAM_DISK}, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, RAM_DISK);
+ }
+
+ @Test (timeout=300000)
+ public void testFallbackToDisk() throws IOException {
+ startUpCluster(REPL_FACTOR, null, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, DEFAULT);
+ }
+
+ /**
+ * If the only available storage is RAM_DISK and the LAZY_PERSIST flag is not
+ * specified, then block placement should fail.
+ *
+ * @throws IOException
+ */
+ @Test (timeout=300000)
+ public void testRamDiskNotChosenByDefault() throws IOException {
+ startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, RAM_DISK}, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ try {
+ makeTestFile(path, BLOCK_SIZE, false);
+ fail("Block placement to RAM_DISK should have failed without lazyPersist flag");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ }
+ }
+
+ @Test (timeout=300000)
+ public void testAppendIsDenied() throws IOException {
+ startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ makeTestFile(path, BLOCK_SIZE, true);
+
+ try {
+ client.append(path.toString(), BUFFER_LENGTH, null, null).close();
+ fail("Append to LazyPersist file did not fail as expected");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ }
+ }
+
+ @Test (timeout=300000)
+ public void testLazyPersistBlocksAreSaved()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR, new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ // Create a test file
+ makeTestFile(path, BLOCK_SIZE * 10, true);
+ LocatedBlocks locatedBlocks = ensureFileReplicasOnStorageType(path, RAM_DISK);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ // Make sure that there is a saved copy of the replica on persistent
+ // storage.
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+
+ final Set<Long> persistedBlockIds = new HashSet<Long>();
+
+ // Make sure at least one non-transient volume has a saved copy of
+ // the replica.
+ for (FsVolumeSpi v : volumes) {
+ if (v.isTransientStorage()) {
+ continue;
+ }
+
+ FsVolumeImpl volume = (FsVolumeImpl) v;
+ File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
+
+ for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+ File persistedBlockFile = new File(lazyPersistDir, "blk_" + lb.getBlock().getBlockId());
+ if (persistedBlockFile.exists()) {
+ // Found a persisted copy for this block!
+ boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
+ assertThat(added, is(true));
+ }
+ }
+ }
+
+ // We should have found a persisted copy for each located block.
+ assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+ }
+
+
+ @Test (timeout=300000)
+ public void testRamDiskEviction()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR,
+ new StorageType[] {RAM_DISK, DEFAULT },
+ (2 * BLOCK_SIZE - 1)); // 1 replica + delta.
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+
+ makeTestFile(path1, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job.
+ // However the block replica should not be evicted from RAM_DISK yet.
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ // Create another file with a replica on RAM_DISK.
+ makeTestFile(path2, BLOCK_SIZE, true);
+ DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
+ Thread.sleep(10 * 1000);
+
+ // Make sure that the second file's block replica is on RAM_DISK, whereas
+ // the original file's block replica is now on disk.
+ ensureFileReplicasOnStorageType(path2, RAM_DISK);
+ ensureFileReplicasOnStorageType(path1, DEFAULT);
+ }
+
+ /**
+ * TODO: Stub test, to be completed.
+ * Verify that checksum computation is skipped for files written to memory.
+ */
+ @Test (timeout=300000)
+ public void testChecksumIsSkipped()
+ throws IOException, InterruptedException {
+ startUpCluster(REPL_FACTOR,
+ new StorageType[] {RAM_DISK, DEFAULT }, -1);
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+
+ makeTestFile(path1, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path1, RAM_DISK);
+
+ // Verify checksum was not computed.
+
+ }
+
+ // ---- Utility functions for all test cases -------------------------------
+
+ /**
+ * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially
+ * capped. If tmpfsStorageLimit < 0 then it is ignored.
+ */
+ private void startUpCluster(final int numDataNodes,
+ final StorageType[] storageTypes,
+ final long ramDiskStorageLimit)
+ throws IOException {
+
+ conf = new Configuration();
+ conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC,
+ LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
+ conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
+ conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ HEARTBEAT_RECHECK_INTERVAL_MSEC);
+ conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
+ LAZY_WRITER_INTERVAL_SEC);
+
+ REPL_FACTOR = 1; //Reset if case a test has modified the value
+
+ cluster = new MiniDFSCluster
+ .Builder(conf)
+ .numDataNodes(numDataNodes)
+ .storageTypes(storageTypes != null ? storageTypes : new StorageType[] { DEFAULT, DEFAULT })
+ .build();
+ fs = cluster.getFileSystem();
+ client = fs.getClient();
+
+ // Artifically cap the storage capacity of the tmpfs volume.
+ if (ramDiskStorageLimit >= 0) {
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+
+ for (FsVolumeSpi volume : volumes) {
+ if (volume.getStorageType() == RAM_DISK) {
+ ((FsTransientVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
+ }
+ }
+ }
+
+ LOG.info("Cluster startup complete");
+ }
+
+ private void makeTestFile(Path path, long length, final boolean isLazyPersist)
+ throws IOException {
+
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+
+ if (isLazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+
+
+ FSDataOutputStream fos = null;
+
+ try {
+ fos =
+ fs.create(path,
+ FsPermission.getFileDefault(),
+ createFlags,
+ BUFFER_LENGTH,
+ REPL_FACTOR,
+ BLOCK_SIZE,
+ null);
+
+ // Allocate a block.
+ byte[] buffer = new byte[BUFFER_LENGTH];
+ for (int bytesWritten = 0; bytesWritten < length; ) {
+ fos.write(buffer, 0, buffer.length);
+ bytesWritten += buffer.length;
+ }
+ if (length > 0) {
+ fos.hsync();
+ }
+ } finally {
+ IOUtils.closeQuietly(fos);
+ }
+ }
+
+ private LocatedBlocks ensureFileReplicasOnStorageType(
+ Path path, StorageType storageType) throws IOException {
+ // Ensure that returned block locations returned are correct!
+ long fileLength = client.getFileInfo(path.toString()).getLen();
+ LocatedBlocks locatedBlocks =
+ client.getLocatedBlocks(path.toString(), 0, fileLength);
+ for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+ assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
+ }
+
+ return locatedBlocks;
+ }
+}
[34/34] git commit: HDFS-7171. Fix Jenkins failures in HDFS-6581
branch. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-7171. Fix Jenkins failures in HDFS-6581 branch. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/137f0b8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/137f0b8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/137f0b8c
Branch: refs/heads/branch-2.6
Commit: 137f0b8cbfd496cae2750f1bf219d2e6a84b9fe2
Parents: e6457a0
Author: arp <ar...@apache.org>
Authored: Tue Sep 30 18:25:20 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:53 2014 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 1 -
.../hadoop/hdfs/TestBlockStoragePolicy.java | 12 ++++-
.../hdfs/server/mover/TestStorageMover.java | 52 --------------------
3 files changed, 11 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/137f0b8c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3639b2f..639faf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2356,7 +2356,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
src = FSDirectory.resolvePath(src, pathComponents, dir);
- INode inode = dir.getINode(src);
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = blockManager.getStoragePolicy(policyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/137f0b8c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index e89d406..cf032a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -71,6 +71,7 @@ public class TestBlockStoragePolicy {
static final byte HOT = HdfsConstants.HOT_STORAGE_POLICY_ID;
static final byte ONESSD = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
static final byte ALLSSD = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
+ static final byte LAZY_PERSIST = HdfsConstants.LAZY_PERSIST_STORAGE_POLICY_ID;
@Test (timeout=300000)
public void testConfigKeyEnabled() throws IOException {
@@ -126,6 +127,9 @@ public class TestBlockStoragePolicy {
expectedPolicyStrings.put(ALLSSD, "BlockStoragePolicy{ALL_SSD:" + ALLSSD +
", storageTypes=[SSD], creationFallbacks=[DISK], " +
"replicationFallbacks=[DISK]}");
+ expectedPolicyStrings.put(LAZY_PERSIST,
+ "BlockStoragePolicy{LAZY_PERSIST:" + LAZY_PERSIST + ", storageTypes=[RAM_DISK, DISK], " +
+ "creationFallbacks=[DISK], replicationFallbacks=[DISK]}");
for(byte i = 1; i < 16; i++) {
final BlockStoragePolicy policy = POLICY_SUITE.getPolicy(i);
@@ -1151,13 +1155,19 @@ public class TestBlockStoragePolicy {
final DistributedFileSystem fs = cluster.getFileSystem();
try {
BlockStoragePolicy[] policies = fs.getStoragePolicies();
- Assert.assertEquals(5, policies.length);
+ Assert.assertEquals(6, policies.length);
Assert.assertEquals(POLICY_SUITE.getPolicy(COLD).toString(),
policies[0].toString());
Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
policies[1].toString());
Assert.assertEquals(POLICY_SUITE.getPolicy(HOT).toString(),
policies[2].toString());
+ Assert.assertEquals(POLICY_SUITE.getPolicy(ONESSD).toString(),
+ policies[3].toString());
+ Assert.assertEquals(POLICY_SUITE.getPolicy(ALLSSD).toString(),
+ policies[4].toString());
+ Assert.assertEquals(POLICY_SUITE.getPolicy(LAZY_PERSIST).toString(),
+ policies[5].toString());
} finally {
IOUtils.cleanup(null, fs);
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/137f0b8c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index b690165..0425dc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -783,56 +783,4 @@ public class TestStorageMover {
test.shutdownCluster();
}
}
-
- /**
- * Test blocks of lazy_persist file on RAM_DISK will not be moved to other
- * storage types by the Storage Mover.
- */
- @Test
- public void testRamDiskNotMoved() throws Exception {
- LOG.info("testRamDiskNotMoved");
- final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
- final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
-
- final long diskCapacity = 100 * BLOCK_SIZE;
- final long archiveCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
- * BLOCK_SIZE;
- final long ramDiskCapacity = 10 * BLOCK_SIZE;
- final long[][] capacities = genCapacities(1, 0, 0, 1,
- diskCapacity, archiveCapacity, ramDiskCapacity);
- final int LAZY_WRITER_INTERVAL_SEC = 1;
- final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
- 1, (short)1, genStorageTypes(1, 0, 0, 1), capacities);
- clusterScheme.conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
- LAZY_WRITER_INTERVAL_SEC);
- final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
-
- try {
- test.runBasicTest(false);
-
- // test creating a hot RAM_DISK file
- final int SEED = 0xFADED;
- final Path foo_hot = new Path(pathPolicyMap.hot, "foo_hot");
- DFSTestUtil.createFile(test.dfs, foo_hot, true, BLOCK_SIZE, BLOCK_SIZE,
- BLOCK_SIZE, (short) 1, SEED, true);
- Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
- test.dfs.getClient(), foo_hot, StorageType.RAM_DISK));
-
- // Sleep for a short time to allow the lazy writer thread to do its job
- Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
-
- // Verify policy related name change is allowed
- final Path foo_hot_new = new Path(pathPolicyMap.warm, "foo_hot");
- test.dfs.rename(foo_hot, pathPolicyMap.warm);
- Assert.assertTrue(test.dfs.exists(foo_hot_new));
-
- // Verify blocks on ram disk will not be moved to other storage types by
- // policy based Storage Mover.
- test.migrate();
- Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
- test.dfs.getClient(), foo_hot_new, StorageType.RAM_DISK));
- } finally {
- test.shutdownCluster();
- }
- }
}
[29/34] git commit: HDFS-7143. Fix findbugs warnings in HDFS-6581
branch. (Contributed by Tsz Wo Nicholas Sze)
Posted by ji...@apache.org.
HDFS-7143. Fix findbugs warnings in HDFS-6581 branch. (Contributed by Tsz Wo Nicholas Sze)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05cfd688
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05cfd688
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05cfd688
Branch: refs/heads/branch-2.6
Commit: 05cfd68863737587984f51d36b3d21e4707b9d33
Parents: 9de4d18
Author: arp <ar...@apache.org>
Authored: Wed Sep 24 20:13:30 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:52 2014 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/BlockPoolSlice.java | 50 +++++++++++++++-----
1 file changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05cfd688/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index bfa1772..3f58d38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -29,6 +29,8 @@ import java.io.RandomAccessFile;
import java.util.Scanner;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DU;
import org.apache.hadoop.fs.FileUtil;
@@ -43,6 +45,7 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.datanode.ReplicaWaitingToBeRecovered;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
@@ -57,6 +60,8 @@ import org.apache.hadoop.util.Time;
* This class is synchronized by {@link FsVolumeImpl}.
*/
class BlockPoolSlice {
+ static final Log LOG = LogFactory.getLog(BlockPoolSlice.class);
+
private final String bpid;
private final FsVolumeImpl volume; // volume to which this BlockPool belongs to
private final File currentDir; // StorageDirectory/current/bpid/current
@@ -375,22 +380,36 @@ class BlockPoolSlice {
File targetDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId);
if (blockFile.exists()) {
- File targetBlockFile = new File(targetDir, blockFile.getName());
- File targetMetaFile = new File(targetDir, metaFile.getName());
if (!targetDir.exists() && !targetDir.mkdirs()) {
- FsDatasetImpl.LOG.warn("Failed to move " + blockFile + " to " + targetDir);
+ LOG.warn("Failed to mkdirs " + targetDir);
+ continue;
+ }
+
+ final File targetMetaFile = new File(targetDir, metaFile.getName());
+ try {
+ NativeIO.renameTo(metaFile, targetMetaFile);
+ } catch (IOException e) {
+ LOG.warn("Failed to move meta file from "
+ + metaFile + " to " + targetMetaFile, e);
continue;
+
}
- metaFile.renameTo(targetMetaFile);
- blockFile.renameTo(targetBlockFile);
+ final File targetBlockFile = new File(targetDir, blockFile.getName());
+ try {
+ NativeIO.renameTo(blockFile, targetBlockFile);
+ } catch (IOException e) {
+ LOG.warn("Failed to move block file from "
+ + blockFile + " to " + targetBlockFile, e);
+ continue;
+ }
if (targetBlockFile.exists() && targetMetaFile.exists()) {
++numRecovered;
} else {
// Failure should be rare.
- FsDatasetImpl.LOG.warn("Failed to move " + blockFile + " to " + targetDir);
+ LOG.warn("Failed to move " + blockFile + " to " + targetDir);
}
}
}
@@ -544,16 +563,23 @@ class BlockPoolSlice {
replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep
+ + ". Will try to delete " + replicaToDelete);
+ }
+
// Update volumeMap.
volumeMap.add(bpid, replicaToKeep);
// Delete the files on disk. Failure here is okay.
- replicaToDelete.getBlockFile().delete();
- replicaToDelete.getMetaFile().delete();
-
- FsDatasetImpl.LOG.info(
- "resolveDuplicateReplicas keeping " + replicaToKeep.getBlockFile() +
- ", deleting " + replicaToDelete.getBlockFile());
+ final File blockFile = replicaToDelete.getBlockFile();
+ if (!blockFile.delete()) {
+ LOG.warn("Failed to delete block file " + blockFile);
+ }
+ final File metaFile = replicaToDelete.getMetaFile();
+ if (!metaFile.delete()) {
+ LOG.warn("Failed to delete meta file " + metaFile);
+ }
return replicaToKeep;
}
[05/34] git commit: HDFS-6926. DN support for saving replicas to
persistent storage and evicting in-memory replicas. (Arpit Agarwal)
Posted by ji...@apache.org.
HDFS-6926. DN support for saving replicas to persistent storage and evicting in-memory replicas. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e01a89b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e01a89b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e01a89b
Branch: refs/heads/branch-2.6
Commit: 8e01a89b618130b9757b63c1b9da657c3f74b91b
Parents: 34d0088
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 15:36:48 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 16:00:49 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../server/datanode/BlockPoolSliceScanner.java | 2 +-
.../hdfs/server/datanode/DataStorage.java | 1 +
.../hdfs/server/datanode/ReplicaInPipeline.java | 2 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 6 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 3 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 45 ++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 312 ++++++++++++++++++-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 6 +
.../fsdataset/impl/LazyWriteReplicaTracker.java | 177 +++++++++++
.../server/datanode/SimulatedFSDataset.java | 5 +
.../hdfs/server/namenode/TestAddBlockRetry.java | 1 +
12 files changed, 548 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 5eb449a..e12b01a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -127,6 +127,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume";
public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+ public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec";
+ public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60;
public static final String DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT =
"dfs.namenode.path.based.cache.block.map.allocation.percent";
public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f;
@@ -229,6 +231,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final float DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT = 2.0f;
public static final String DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS = "dfs.namenode.edit.log.autoroll.check.interval.ms";
public static final int DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS_DEFAULT = 5*60*1000;
+
+ public static final String DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC = "dfs.namenode.lazypersist.file.scrub.interval.sec";
+ public static final int DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT = 5 * 60;
public static final String DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH = "dfs.namenode.edits.noeditlogchannelflush";
public static final boolean DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH_DEFAULT = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
index bbb67fc..61f1e7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
@@ -191,7 +191,7 @@ class BlockPoolSliceScanner {
+ hours + " hours for block pool " + bpid);
// get the list of blocks and arrange them in random order
- List<FinalizedReplica> arr = dataset.getFinalizedBlocks(blockPoolId);
+ List<FinalizedReplica> arr = dataset.getFinalizedBlocksOnPersistentStorage(blockPoolId);
Collections.shuffle(arr);
long scanTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 1508be3..ce33d42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -68,6 +68,7 @@ public class DataStorage extends Storage {
final static String STORAGE_DIR_DETACHED = "detach";
public final static String STORAGE_DIR_RBW = "rbw";
public final static String STORAGE_DIR_FINALIZED = "finalized";
+ public final static String STORAGE_DIR_LAZY_PERSIST = "lazypersist";
public final static String STORAGE_DIR_TMP = "tmp";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
index 08395aa..45862ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
@@ -250,7 +250,7 @@ public class ReplicaInPipeline extends ReplicaInfo
}
}
} else {
- // for create, we can use the requested checksum
+ // for create, we can use the requested checksum
checksum = requestedChecksum;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index 235cd7b..d0d36ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -138,8 +138,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
if (mostAvailableAmongLowVolumes < replicaSize ||
random.nextFloat() < scaledPreferencePercent) {
volume = roundRobinPolicyHighAvailable.chooseVolume(
- highAvailableVolumes,
- replicaSize);
+ highAvailableVolumes, replicaSize);
if (LOG.isDebugEnabled()) {
LOG.debug("Volumes are imbalanced. Selecting " + volume +
" from high available space volumes for write of block size "
@@ -147,8 +146,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
}
} else {
volume = roundRobinPolicyLowAvailable.chooseVolume(
- lowAvailableVolumes,
- replicaSize);
+ lowAvailableVolumes, replicaSize);
if (LOG.isDebugEnabled()) {
LOG.debug("Volumes are imbalanced. Selecting " + volume +
" from low available space volumes for write of block size "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 4a5580c..2bb2e7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -116,6 +116,9 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
/** @return a list of finalized blocks for the given block pool. */
public List<FinalizedReplica> getFinalizedBlocks(String bpid);
+ /** @return a list of finalized blocks for the given block pool. */
+ public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid);
+
/**
* Check whether the in-memory block record matches the block on the disk,
* and, in case that they are not matched, update the record or mark it
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 7839bec..179f3ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.util.Scanner;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DU;
import org.apache.hadoop.fs.FileUtil;
@@ -61,6 +62,7 @@ class BlockPoolSlice {
private final File currentDir; // StorageDirectory/current/bpid/current
// directory where finalized replicas are stored
private final File finalizedDir;
+ private final File lazypersistDir;
private final File rbwDir; // directory store RBW replica
private final File tmpDir; // directory store Temporary replica
private static final String DU_CACHE_FILE = "dfsUsed";
@@ -85,12 +87,24 @@ class BlockPoolSlice {
this.currentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT);
this.finalizedDir = new File(
currentDir, DataStorage.STORAGE_DIR_FINALIZED);
+ this.lazypersistDir = new File(currentDir, DataStorage.STORAGE_DIR_LAZY_PERSIST);
if (!this.finalizedDir.exists()) {
if (!this.finalizedDir.mkdirs()) {
throw new IOException("Failed to mkdirs " + this.finalizedDir);
}
}
+ // Delete all checkpointed replicas on startup.
+ // TODO: We can move checkpointed replicas to the finalized dir and delete
+ // the copy on RAM_DISK. For now we take the simpler approach.
+
+ FileUtil.fullyDelete(lazypersistDir);
+ if (!this.lazypersistDir.exists()) {
+ if (!this.lazypersistDir.mkdirs()) {
+ throw new IOException("Failed to mkdirs " + this.lazypersistDir);
+ }
+ }
+
// Files that were being written when the datanode was last shutdown
// are now moved back to the data directory. It is possible that
// in the future, we might want to do some sort of datanode-local
@@ -142,6 +156,10 @@ class BlockPoolSlice {
return finalizedDir;
}
+ File getLazypersistDir() {
+ return lazypersistDir;
+ }
+
File getRbwDir() {
return rbwDir;
}
@@ -258,12 +276,37 @@ class BlockPoolSlice {
dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length());
return blockFile;
}
-
+
+ File lazyPersistReplica(Block b, File f) throws IOException {
+ File blockFile = FsDatasetImpl.copyBlockFiles(b, f, lazypersistDir);
+ File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
+ dfsUsage.incDfsUsed(b.getNumBytes() + metaFile.length());
+ return blockFile;
+ }
+
+ /**
+ * Move a persisted replica from lazypersist directory to a subdirectory
+ * under finalized.
+ */
+ File activateSavedReplica(Block b, File blockFile) throws IOException {
+ final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId());
+ final File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
+ final File targetBlockFile = new File(blockDir, blockFile.getName());
+ final File targetMetaFile = new File(blockDir, metaFile.getName());
+ FileUtils.moveFile(blockFile, targetBlockFile);
+ FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile);
+ FileUtils.moveFile(metaFile, targetMetaFile);
+ FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile);
+ return targetBlockFile;
+ }
+
void checkDirs() throws DiskErrorException {
DiskChecker.checkDirs(finalizedDir);
DiskChecker.checkDir(tmpDir);
DiskChecker.checkDir(rbwDir);
}
+
+
void getVolumeMap(ReplicaMap volumeMap) throws IOException {
// add finalized replicas
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 701207d..1abc6f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.*;
import java.util.concurrent.Executor;
import javax.management.NotCompliantMBeanException;
@@ -95,6 +96,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
@@ -118,7 +120,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
-
@Override // FsDatasetSpi
public List<FsVolumeImpl> getVolumes() {
return volumes.volumes;
@@ -211,11 +212,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final FsVolumeList volumes;
final Map<String, DatanodeStorage> storageMap;
final FsDatasetAsyncDiskService asyncDiskService;
+ final Daemon lazyWriter;
final FsDatasetCache cacheManager;
private final Configuration conf;
private final int validVolsRequired;
+ private volatile boolean fsRunning;
final ReplicaMap volumeMap;
+ final LazyWriteReplicaTracker lazyWriteReplicaTracker;
+
+ private static final int MAX_BLOCK_EVICTIONS_PER_ITERATION = 3;
+
// Used for synchronizing access to usage stats
private final Object statsLock = new Object();
@@ -225,6 +232,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
FsDatasetImpl(DataNode datanode, DataStorage storage, Configuration conf
) throws IOException {
+ this.fsRunning = true;
this.datanode = datanode;
this.dataStorage = storage;
this.conf = conf;
@@ -255,6 +263,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
storageMap = new ConcurrentHashMap<String, DatanodeStorage>();
volumeMap = new ReplicaMap(this);
+ lazyWriteReplicaTracker = new LazyWriteReplicaTracker(this);
+
@SuppressWarnings("unchecked")
final VolumeChoosingPolicy<FsVolumeImpl> blockChooserImpl =
ReflectionUtils.newInstance(conf.getClass(
@@ -264,11 +274,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumes = new FsVolumeList(volsFailed, blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode);
+ // TODO: Initialize transientReplicaTracker from blocks on disk.
+
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
addVolume(dataLocations, storage.getStorageDir(idx));
}
cacheManager = new FsDatasetCache(this);
+ lazyWriter = new Daemon(new LazyWriter(
+ conf.getInt(DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
+ DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC)));
+ lazyWriter.start();
registerMBean(datanode.getDatanodeUuid());
}
@@ -664,8 +680,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
return new ReplicaInputStreams(blockInFile.getFD(), metaInFile.getFD());
}
- static File moveBlockFiles(Block b, File srcfile, File destdir
- ) throws IOException {
+ static File moveBlockFiles(Block b, File srcfile, File destdir)
+ throws IOException {
final File dstfile = new File(destdir, b.getBlockName());
final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp());
final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp());
@@ -688,6 +704,30 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
return dstfile;
}
+ static File copyBlockFiles(Block b, File srcfile, File destdir)
+ throws IOException {
+ final File dstfile = new File(destdir, b.getBlockName());
+ final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp());
+ final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp());
+ try {
+ FileUtils.copyFile(srcmeta, dstmeta);
+ } catch (IOException e) {
+ throw new IOException("Failed to copy meta file for " + b
+ + " from " + srcmeta + " to " + dstmeta, e);
+ }
+ try {
+ FileUtils.copyFile(srcfile, dstfile);
+ } catch (IOException e) {
+ throw new IOException("Failed to copy block file for " + b
+ + " from " + srcfile + " to " + dstfile.getAbsolutePath(), e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("addBlock: Moved " + srcmeta + " to " + dstmeta
+ + " and " + srcfile + " to " + dstfile);
+ }
+ return dstfile;
+ }
+
static private void truncateBlock(File blockFile, File metaFile,
long oldlen, long newlen) throws IOException {
LOG.info("truncateBlock: blockFile=" + blockFile
@@ -950,6 +990,83 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
+ /**
+ * Attempt to evict one or more transient block replicas we have at least
+ * spaceNeeded bytes free.
+ *
+ * @return true if we were able to free up at least spaceNeeded bytes, false
+ * otherwise.
+ */
+ private boolean tryToEvictBlocks(final String bpid, final long spaceNeeded)
+ throws IOException {
+
+ boolean isAvailable = false;
+
+ LOG.info("Attempting to evict blocks from transient storage");
+
+ // Reverse the map so we can iterate in order of replica creation times,
+ // evicting oldest replicas one at a time until we have sufficient space.
+ TreeMultimap<Long, LazyWriteReplicaTracker.ReplicaState> lruMap =
+ lazyWriteReplicaTracker.getLruMap();
+ int blocksEvicted = 0;
+
+ // TODO: It is really inefficient to do this with the Object lock held!
+ // TODO: This logic is here just for prototyping.
+ // TODO: We should replace it with proactive discard when ram_disk free space
+ // TODO: falls below a low watermark. That way we avoid fs operations on the
+ // TODO: hot path with the lock held.
+ synchronized (this) {
+ long currentTime = System.currentTimeMillis() / 1000;
+ for (Map.Entry<Long, LazyWriteReplicaTracker.ReplicaState> entry : lruMap.entries()) {
+ LazyWriteReplicaTracker.ReplicaState lazyWriteReplica = entry.getValue();
+ LOG.info("RAM_DISK: Evicting blockId=" + lazyWriteReplica.blockId +
+ "; block LMT=" + entry.getKey() +
+ "; currentTime=" + currentTime);
+ ReplicaInfo replicaInfo = getReplicaInfo(bpid, lazyWriteReplica.blockId);
+ Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
+ File blockFile = replicaInfo.getBlockFile();
+ File metaFile = replicaInfo.getMetaFile();
+ long used = blockFile.length() + metaFile.length();
+ lazyWriteReplicaTracker.discardReplica(bpid, entry.getValue().blockId, false);
+
+ // Move the persisted replica to the finalized directory of
+ // the target volume.
+ BlockPoolSlice bpSlice =
+ lazyWriteReplica.lazyPersistVolume.getBlockPoolSlice(bpid);
+ File newBlockFile = bpSlice.activateSavedReplica(
+ replicaInfo, lazyWriteReplica.savedBlockFile);
+
+ ReplicaInfo newReplicaInfo =
+ new FinalizedReplica(replicaInfo.getBlockId(),
+ replicaInfo.getBytesOnDisk(),
+ replicaInfo.getGenerationStamp(),
+ lazyWriteReplica.lazyPersistVolume,
+ newBlockFile.getParentFile());
+
+ // Update the volumeMap entry. This removes the old entry.
+ volumeMap.add(bpid, newReplicaInfo);
+
+ // Remove the old replicas.
+ blockFile.delete();
+ metaFile.delete();
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, used);
+ ++blocksEvicted;
+
+ if (replicaInfo.getVolume().getAvailable() > spaceNeeded) {
+ LOG.info("RAM_DISK: freed up " + spaceNeeded + " bytes for new block");
+ isAvailable = true;
+ break;
+ }
+
+ if (blocksEvicted == MAX_BLOCK_EVICTIONS_PER_ITERATION) {
+ break;
+ }
+ }
+ }
+
+ return isAvailable;
+ }
+
@Override // FsDatasetSpi
public synchronized ReplicaInPipeline createRbw(StorageType storageType,
ExtendedBlock b, boolean allowLazyPersist) throws IOException {
@@ -972,7 +1089,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
} catch (DiskOutOfSpaceException de) {
if (allowLazyPersist) {
- allowLazyPersist = false;
+ if (!tryToEvictBlocks(b.getBlockPoolId(), b.getNumBytes())) {
+ // Eviction did not work, we'll just fallback to DEFAULT storage.
+ LOG.info("RAM_DISK: Failed to free up " + b.getNumBytes() +
+ " bytes for new block. Will fallback to DEFAULT " +
+ "storage");
+ allowLazyPersist = false;
+ }
continue;
}
throw de;
@@ -984,6 +1107,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(),
b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
+
return newReplicaInfo;
}
@@ -1129,7 +1253,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(),
b.getGenerationStamp(), v, f.getParentFile(), 0);
volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
-
return newReplicaInfo;
}
@@ -1196,8 +1319,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File dest = v.addFinalizedBlock(
bpid, replicaInfo, f, replicaInfo.getBytesReserved());
newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile());
+
+ if (v.isTransientStorage()) {
+ lazyWriteReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
+
+ // Schedule a checkpoint.
+ ((LazyWriter) lazyWriter.getRunnable())
+ .addReplicaToLazyWriteQueue(bpid, replicaInfo.getBlockId());
+ }
}
volumeMap.add(bpid, newReplicaInfo);
+
return newReplicaInfo;
}
@@ -1217,6 +1349,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
replicaInfo.getMetaFile(), b.getLocalBlock())) {
LOG.warn("Block " + b + " unfinalized and removed. " );
}
+ if (replicaInfo.getVolume().isTransientStorage()) {
+ lazyWriteReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true);
+ }
}
}
@@ -1313,6 +1448,22 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
/**
+ * Get the list of finalized blocks from in-memory blockmap for a block pool.
+ */
+ @Override
+ public synchronized List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) {
+ ArrayList<FinalizedReplica> finalized =
+ new ArrayList<FinalizedReplica>(volumeMap.size(bpid));
+ for (ReplicaInfo b : volumeMap.replicas(bpid)) {
+ if(!b.getVolume().isTransientStorage() &&
+ b.getState() == ReplicaState.FINALIZED) {
+ finalized.add(new FinalizedReplica((FinalizedReplica)b));
+ }
+ }
+ return finalized;
+ }
+
+ /**
* Check whether the given block is a valid one.
* valid means finalized
*/
@@ -1429,6 +1580,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumeMap.remove(bpid, invalidBlks[i]);
}
+ if (v.isTransientStorage()) {
+ lazyWriteReplicaTracker.discardReplica(bpid, invalidBlks[i].getBlockId(), true);
+ }
+
// If a DFSClient has the replica in its cache of short-circuit file
// descriptors (and the client is using ShortCircuitShm), invalidate it.
datanode.getShortCircuitRegistry().processBlockInvalidation(
@@ -1649,8 +1804,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public void shutdown() {
- if (mbeanName != null)
+ fsRunning = false;
+
+ ((LazyWriter) lazyWriter.getRunnable()).stop();
+ lazyWriter.interrupt();
+
+ if (mbeanName != null) {
MBeans.unregister(mbeanName);
+ }
if (asyncDiskService != null) {
asyncDiskService.shutdown();
@@ -1659,6 +1820,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if(volumes != null) {
volumes.shutdown();
}
+
+ try {
+ lazyWriter.join();
+ } catch (InterruptedException ie) {
+ LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
+ "from LazyWriter.join");
+ }
}
@Override // FSDatasetMBean
@@ -1691,7 +1859,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override
public void checkAndUpdate(String bpid, long blockId, File diskFile,
- File diskMetaFile, FsVolumeSpi vol) {
+ File diskMetaFile, FsVolumeSpi vol) throws IOException {
Block corruptBlock = null;
ReplicaInfo memBlockInfo;
synchronized (this) {
@@ -1724,6 +1892,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (blockScanner != null) {
blockScanner.deleteBlock(bpid, new Block(blockId));
}
+ if (vol.isTransientStorage()) {
+ lazyWriteReplicaTracker.discardReplica(bpid, blockId, true);
+ }
LOG.warn("Removed block " + blockId
+ " from memory with missing block file on the disk");
// Finally remove the metadata file
@@ -1747,6 +1918,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (blockScanner != null) {
blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo));
}
+ if (vol.isTransientStorage()) {
+ lazyWriteReplicaTracker.addReplica(bpid, blockId, (FsVolumeImpl) vol);
+ }
LOG.warn("Added missing block to memory " + diskBlockInfo);
return;
}
@@ -1924,9 +2098,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final String bpid = oldBlock.getBlockPoolId();
final ReplicaInfo replica = volumeMap.get(bpid, oldBlock.getBlockId());
LOG.info("updateReplica: " + oldBlock
- + ", recoveryId=" + recoveryId
- + ", length=" + newlength
- + ", replica=" + replica);
+ + ", recoveryId=" + recoveryId
+ + ", length=" + newlength
+ + ", replica=" + replica);
//check replica
if (replica == null) {
@@ -2196,5 +2370,123 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
asyncDiskService.submitSyncFileRangeRequest(fsVolumeImpl, fd, offset,
nbytes, flags);
}
+
+ private static class BlockIdPair {
+ final String bpid;
+ final long blockId;
+
+ BlockIdPair(final String bpid, final long blockId) {
+ this.bpid = bpid;
+ this.blockId = blockId;
+ }
+ }
+
+ private class LazyWriter implements Runnable {
+ private volatile boolean shouldRun = true;
+ final int checkpointerInterval;
+
+ final private Queue<BlockIdPair> blocksPendingCheckpoint;
+
+ public LazyWriter(final int checkpointerInterval) {
+ this.checkpointerInterval = checkpointerInterval;
+ blocksPendingCheckpoint = new LinkedList<BlockIdPair>();
+ }
+
+ // Schedule a replica for writing to persistent storage.
+ public synchronized void addReplicaToLazyWriteQueue(
+ String bpid, long blockId) {
+ LOG.info("Block with blockId=" + blockId + "; bpid=" + bpid + " added to lazy writer queue");
+ blocksPendingCheckpoint.add(new BlockIdPair(bpid, blockId));
+ }
+
+ private void moveReplicaToNewVolume(String bpid, long blockId)
+ throws IOException {
+
+ LOG.info("LazyWriter invoked to save blockId=" + blockId + "; bpid=" + bpid);
+
+ FsVolumeImpl targetVolume = null;
+ Block block = null;
+ File blockFile = null;
+
+ synchronized (this) {
+ block = getStoredBlock(bpid, blockId);
+ blockFile = getFile(bpid, blockId);
+
+ if (block == null) {
+ // The block was deleted before it could be checkpointed.
+ return;
+ }
+
+ // Pick a target volume for the block.
+ targetVolume = volumes.getNextVolume(
+ StorageType.DEFAULT, block.getNumBytes());
+ }
+
+ LOG.info("LazyWriter starting to save blockId=" + blockId + "; bpid=" + bpid);
+ lazyWriteReplicaTracker.recordStartLazyPersist(bpid, blockId, targetVolume);
+ File savedBlockFile = targetVolume.getBlockPoolSlice(bpid)
+ .lazyPersistReplica(block, blockFile);
+ lazyWriteReplicaTracker.recordEndLazyPersist(bpid, blockId, savedBlockFile);
+ LOG.info("LazyWriter finished saving blockId=" + blockId + "; bpid=" + bpid +
+ " to file " + savedBlockFile);
+ }
+
+ /**
+ * Checkpoint a pending replica to persistent storage now.
+ * @return true if there is more work to be done, false otherwise.
+ */
+ private boolean saveNextReplica() {
+ BlockIdPair blockIdPair = null;
+ int moreWorkThreshold = 0;
+
+ try {
+ synchronized (this) {
+ // Dequeue the next replica waiting to be checkpointed.
+ blockIdPair = blocksPendingCheckpoint.poll();
+ if (blockIdPair == null) {
+ LOG.info("LazyWriter has no blocks to persist. " +
+ "Thread going to sleep.");
+ return false;
+ }
+ }
+
+ // Move the replica outside the lock.
+ moveReplicaToNewVolume(blockIdPair.bpid, blockIdPair.blockId);
+
+ } catch(IOException ioe) {
+ // If we failed, put the block on the queue and let a retry
+ // interval elapse before we try again so we don't try to keep
+ // checkpointing the same block in a tight loop.
+ synchronized (this) {
+ blocksPendingCheckpoint.add(blockIdPair);
+ ++moreWorkThreshold;
+ }
+ }
+
+ synchronized (this) {
+ return blocksPendingCheckpoint.size() > moreWorkThreshold;
+ }
+ }
+
+ @Override
+ public void run() {
+ while (fsRunning && shouldRun) {
+ try {
+ if (!saveNextReplica()) {
+ Thread.sleep(checkpointerInterval * 1000);
+ }
+ } catch (InterruptedException e) {
+ LOG.info("LazyWriter was interrupted, exiting");
+ break;
+ } catch (Exception e) {
+ LOG.error("Ignoring exception in LazyWriter:", e);
+ }
+ }
+ }
+
+ public void stop() {
+ shouldRun = false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 4c0b5f8..994344e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -380,6 +380,8 @@ public class FsVolumeImpl implements FsVolumeSpi {
File bpCurrentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT);
File finalizedDir = new File(bpCurrentDir,
DataStorage.STORAGE_DIR_FINALIZED);
+ File lazypersistDir = new File(bpCurrentDir,
+ DataStorage.STORAGE_DIR_LAZY_PERSIST);
File rbwDir = new File(bpCurrentDir, DataStorage.STORAGE_DIR_RBW);
if (force) {
FileUtil.fullyDelete(bpDir);
@@ -391,6 +393,10 @@ public class FsVolumeImpl implements FsVolumeSpi {
!FileUtil.fullyDelete(finalizedDir)) {
throw new IOException("Failed to delete " + finalizedDir);
}
+ if (!DatanodeUtil.dirNoFilesRecursive(lazypersistDir) ||
+ !FileUtil.fullyDelete(lazypersistDir)) {
+ throw new IOException("Failed to delete " + lazypersistDir);
+ }
FileUtil.fullyDelete(tmpDir);
for (File f : FileUtil.listFiles(bpCurrentDir)) {
if (!f.delete()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
new file mode 100644
index 0000000..ae28f09
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyWriteReplicaTracker.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+class LazyWriteReplicaTracker {
+
+ enum State {
+ IN_MEMORY,
+ LAZY_PERSIST_IN_PROGRESS,
+ LAZY_PERSIST_COMPLETE,
+ }
+
+ static class ReplicaState implements Comparable<ReplicaState> {
+
+ final String bpid;
+ final long blockId;
+ State state;
+
+ /**
+ * transient storage volume that holds the original replica.
+ */
+ final FsVolumeImpl transientVolume;
+
+ /**
+ * Persistent volume that holds or will hold the saved replica.
+ */
+ FsVolumeImpl lazyPersistVolume;
+ File savedBlockFile;
+
+ ReplicaState(final String bpid, final long blockId, FsVolumeImpl transientVolume) {
+ this.bpid = bpid;
+ this.blockId = blockId;
+ this.transientVolume = transientVolume;
+ state = State.IN_MEMORY;
+ lazyPersistVolume = null;
+ savedBlockFile = null;
+ }
+
+ @Override
+ public int hashCode() {
+ return bpid.hashCode() ^ (int) blockId;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ ReplicaState otherState = (ReplicaState) other;
+ return (otherState.bpid.equals(bpid) && otherState.blockId == blockId);
+ }
+
+ @Override
+ public int compareTo(ReplicaState other) {
+ if (blockId == other.blockId) {
+ return 0;
+ } else if (blockId < other.blockId) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ }
+
+ final FsDatasetImpl fsDataset;
+
+ /**
+ * Map of blockpool ID to map of blockID to ReplicaInfo.
+ */
+ final Map<String, Map<Long, ReplicaState>> replicaMaps;
+
+ /**
+ * A map of blockId to persist complete time for transient blocks. This allows
+ * us to evict LRU blocks from transient storage. Protected by 'this'
+ * Object lock.
+ */
+ final Map<ReplicaState, Long> persistTimeMap;
+
+ LazyWriteReplicaTracker(final FsDatasetImpl fsDataset) {
+ this.fsDataset = fsDataset;
+ replicaMaps = new HashMap<String, Map<Long, ReplicaState>>();
+ persistTimeMap = new HashMap<ReplicaState, Long>();
+ }
+
+ TreeMultimap<Long, ReplicaState> getLruMap() {
+ // TODO: This can be made more efficient.
+ TreeMultimap<Long, ReplicaState> reversedMap = TreeMultimap.create();
+ for (Map.Entry<ReplicaState, Long> entry : persistTimeMap.entrySet()) {
+ reversedMap.put(entry.getValue(), entry.getKey());
+ }
+ return reversedMap;
+ }
+
+ synchronized void addReplica(String bpid, long blockId,
+ final FsVolumeImpl transientVolume) {
+ Map<Long, ReplicaState> map = replicaMaps.get(bpid);
+ if (map == null) {
+ map = new HashMap<Long, ReplicaState>();
+ replicaMaps.put(bpid, map);
+ }
+ map.put(blockId, new ReplicaState(bpid, blockId, transientVolume));
+ }
+
+ synchronized void recordStartLazyPersist(
+ final String bpid, final long blockId, FsVolumeImpl checkpointVolume) {
+ Map<Long, ReplicaState> map = replicaMaps.get(bpid);
+ ReplicaState replicaState = map.get(blockId);
+ replicaState.state = State.LAZY_PERSIST_IN_PROGRESS;
+ replicaState.lazyPersistVolume = checkpointVolume;
+ }
+
+ synchronized void recordEndLazyPersist(
+ final String bpid, final long blockId, File savedBlockFile) {
+ Map<Long, ReplicaState> map = replicaMaps.get(bpid);
+ ReplicaState replicaState = map.get(blockId);
+
+ if (replicaState == null) {
+ throw new IllegalStateException("Unknown replica bpid=" +
+ bpid + "; blockId=" + blockId);
+ }
+ replicaState.state = State.LAZY_PERSIST_COMPLETE;
+ replicaState.savedBlockFile = savedBlockFile;
+ persistTimeMap.put(replicaState, System.currentTimeMillis() / 1000);
+ }
+
+ synchronized void discardReplica(
+ final String bpid, final long blockId, boolean force) {
+ Map<Long, ReplicaState> map = replicaMaps.get(bpid);
+ ReplicaState replicaState = map.get(blockId);
+
+ if (replicaState == null) {
+ if (force) {
+ return;
+ }
+ throw new IllegalStateException("Unknown replica bpid=" +
+ bpid + "; blockId=" + blockId);
+ }
+
+ if (replicaState.state != State.LAZY_PERSIST_COMPLETE && !force) {
+ throw new IllegalStateException("Discarding replica without " +
+ "saving it to disk bpid=" + bpid + "; blockId=" + blockId);
+
+ }
+
+ map.remove(blockId);
+ persistTimeMap.remove(replicaState);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index eab599d..312da1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -1122,6 +1122,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override
+ public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public Map<String, Object> getVolumeInfoMap() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01a89b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
index cf37a54..6098ebf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;