You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/08/27 18:04:52 UTC
[1/2] git commit: HDFS-6921. Add LazyPersist flag to FileStatus.
(Arpit Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 073fae5e7 -> fecbf9bb6 (forced update)
HDFS-6921. Add LazyPersist flag to FileStatus. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20a879d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20a879d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20a879d5
Branch: refs/heads/HDFS-6581
Commit: 20a879d538730313a0c4790bbbcbfbf843db89ff
Parents: 7ac5b6e
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 08:52:55 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 09:04:37 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/CreateFlag.java | 16 ++++++++++++++-
.../java/org/apache/hadoop/fs/FileStatus.java | 21 ++++++++++++++++++++
.../apache/hadoop/fs/RawLocalFileSystem.java | 3 ++-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 8 +++++---
.../hadoop/fs/http/server/FSOperations.java | 1 +
.../hadoop-hdfs/CHANGES-HDFS-6581.txt | 4 ++++
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 18 +++++++++++++----
.../hdfs/protocol/HdfsLocatedFileStatus.java | 14 ++++++-------
.../protocol/SnapshottableDirectoryStatus.java | 2 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 9 +++++++++
.../hdfs/server/namenode/FSDirectory.java | 5 +++--
.../hdfs/server/namenode/FSNamesystem.java | 17 ++++++++--------
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 7 ++++---
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 1 +
.../hadoop/hdfs/TestDFSClientRetries.java | 4 ++--
.../java/org/apache/hadoop/hdfs/TestLease.java | 4 ++--
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 ++--
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
19 files changed, 103 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index 252f37b..c5d23b4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -79,7 +79,21 @@ public enum CreateFlag {
/**
* Force closed blocks to disk. Similar to POSIX O_SYNC. See javadoc for description.
*/
- SYNC_BLOCK((short) 0x08);
+ SYNC_BLOCK((short) 0x08),
+
+ /**
+ * Create the block on transient storage (RAM) if available. If
+ * transient storage is unavailable then the block will be created
+ * on disk.
+ *
+ * HDFS will make a best effort to lazily write these files to persistent
+ * storage, however file contents may be lost at any time due to process/
+ * node restarts, hence there is no guarantee of data durability.
+ *
+ * This flag must only be used for intermediate data whose loss can be
+ * tolerated by the application.
+ */
+ LAZY_PERSIST((short) 0x10);
private final short mode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index b261f7f..92b4e38 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -38,6 +38,7 @@ public class FileStatus implements Writable, Comparable {
private boolean isdir;
private short block_replication;
private long blocksize;
+ private boolean isLazyPersist;
private long modification_time;
private long access_time;
private FsPermission permission;
@@ -73,6 +74,18 @@ public class FileStatus implements Writable, Comparable {
FsPermission permission, String owner, String group,
Path symlink,
Path path) {
+ this(length, isdir, block_replication, blocksize, false,
+ modification_time, access_time, permission, owner, group,
+ symlink, path);
+ }
+
+ public FileStatus(long length, boolean isdir,
+ int block_replication,
+ long blocksize, boolean isLazyPersist,
+ long modification_time, long access_time,
+ FsPermission permission, String owner, String group,
+ Path symlink,
+ Path path) {
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
@@ -92,6 +105,7 @@ public class FileStatus implements Writable, Comparable {
this.group = (group == null) ? "" : group;
this.symlink = symlink;
this.path = path;
+ this.isLazyPersist = isLazyPersist;
// The variables isdir and symlink indicate the type:
// 1. isdir implies directory, in which case symlink must be null.
// 2. !isdir implies a file or symlink, symlink != null implies a
@@ -168,6 +182,13 @@ public class FileStatus implements Writable, Comparable {
}
/**
+ * Get whether the file is lazyPersist.
+ */
+ public boolean isLazyPersist() {
+ return isLazyPersist;
+ }
+
+ /**
* Get the replication factor of a file.
* @return the replication factor of a file.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index a06e3a6..c3852df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -762,6 +762,7 @@ public class RawLocalFileSystem extends FileSystem {
false,
fs.getReplication(),
fs.getBlockSize(),
+ fs.isLazyPersist(),
fs.getModificationTime(),
fs.getAccessTime(),
fs.getPermission(),
@@ -777,7 +778,7 @@ public class RawLocalFileSystem extends FileSystem {
* when available.
*/
if (!target.isEmpty()) {
- return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(),
+ return new FileStatus(0, false, 0, 0, false, 0, 0, FsPermission.getDefault(),
"", "", new Path(target), f);
}
// f refers to a file or directory that does not exist
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 3749bc3..f7d2e90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -160,6 +160,7 @@ public class HttpFSFileSystem extends FileSystem
public static final String XATTR_NAME_JSON = "name";
public static final String XATTR_VALUE_JSON = "value";
public static final String XATTRNAMES_JSON = "XAttrNames";
+ public static final String LAZY_PERSIST_JSON = "LazyPersist";
public static final String FILE_CHECKSUM_JSON = "FileChecksum";
public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
@@ -954,19 +955,20 @@ public class HttpFSFileSystem extends FileSystem
long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
+ boolean isLazyPersist = ((Boolean) json.get(LAZY_PERSIST_JSON)).booleanValue();
FileStatus fileStatus = null;
switch (type) {
case FILE:
case DIRECTORY:
fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
- replication, blockSize, mTime, aTime,
- permission, owner, group, path);
+ replication, blockSize, false, mTime, aTime,
+ permission, owner, group, null, path);
break;
case SYMLINK:
Path symLink = null;
fileStatus = new FileStatus(len, false,
- replication, blockSize, mTime, aTime,
+ replication, blockSize, isLazyPersist, mTime, aTime,
permission, owner, group, symLink,
path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index e7d92f5..bcc0476 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -125,6 +125,7 @@ public class FSOperations {
fileStatus.getModificationTime());
json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, fileStatus.getBlockSize());
json.put(HttpFSFileSystem.REPLICATION_JSON, fileStatus.getReplication());
+ json.put(HttpFSFileSystem.LAZY_PERSIST_JSON, fileStatus.isLazyPersist());
if ( (aclStatus != null) && !(aclStatus.getEntries().isEmpty()) ) {
json.put(HttpFSFileSystem.ACL_BIT_JSON,true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
new file mode 100644
index 0000000..706c03a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -0,0 +1,4 @@
+ BREAKDOWN OF HDFS-6581 SUBTASKS AND RELATED JIRAS
+
+ HDFS-6921. Add LazyPersist flag to FileStatus. (Arpit Agarwal)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 3d05639..49c5cde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -39,6 +39,7 @@ public class HdfsFileStatus {
private final boolean isdir;
private final short block_replication;
private final long blocksize;
+ private final boolean isLazyPersist;
private final long modification_time;
private final long access_time;
private final FsPermission permission;
@@ -69,13 +70,15 @@ public class HdfsFileStatus {
* @param feInfo the file's encryption info
*/
public HdfsFileStatus(long length, boolean isdir, int block_replication,
- long blocksize, long modification_time, long access_time,
- FsPermission permission, String owner, String group, byte[] symlink,
- byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo) {
+ long blocksize, boolean isLazyPersist, long modification_time,
+ long access_time, FsPermission permission, String owner,
+ String group, byte[] symlink, byte[] path, long fileId,
+ int childrenNum, FileEncryptionInfo feInfo) {
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
this.blocksize = blocksize;
+ this.isLazyPersist = isLazyPersist;
this.modification_time = modification_time;
this.access_time = access_time;
this.permission = (permission == null) ?
@@ -125,6 +128,13 @@ public class HdfsFileStatus {
}
/**
+ * @return true if the file is lazyPersist.
+ */
+ final public boolean isLazyPersist() {
+ return isLazyPersist;
+ }
+
+ /**
* Get the replication factor of a file.
* @return the replication factor of a file.
*/
@@ -253,7 +263,7 @@ public class HdfsFileStatus {
public final FileStatus makeQualified(URI defaultUri, Path path) {
return new FileStatus(getLen(), isDir(), getReplication(),
- getBlockSize(), getModificationTime(),
+ getBlockSize(), isLazyPersist(), getModificationTime(),
getAccessTime(),
getPermission(), getOwner(), getGroup(),
isSymlink() ? new Path(getSymlink()) : null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
index a78b8bc..f70df34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -55,13 +55,13 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
* @param feInfo file encryption info
*/
public HdfsLocatedFileStatus(long length, boolean isdir,
- int block_replication, long blocksize, long modification_time,
- long access_time, FsPermission permission, String owner, String group,
- byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
- int childrenNum, FileEncryptionInfo feInfo) {
- super(length, isdir, block_replication, blocksize, modification_time,
- access_time, permission, owner, group, symlink, path, fileId,
- childrenNum, feInfo);
+ int block_replication, long blocksize, boolean isLazyPersist,
+ long modification_time, long access_time, FsPermission permission,
+ String owner, String group, byte[] symlink, byte[] path, long fileId,
+ LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo) {
+ super(length, isdir, block_replication, blocksize, isLazyPersist,
+ modification_time, access_time, permission, owner, group, symlink,
+ path, fileId, childrenNum, feInfo);
this.locations = locations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
index d395283..d0cebfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
@@ -59,7 +59,7 @@ public class SnapshottableDirectoryStatus {
FsPermission permission, String owner, String group, byte[] localName,
long inodeId, int childrenNum,
int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
- this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
+ this.dirStatus = new HdfsFileStatus(0, true, 0, 0, false, modification_time,
access_time, permission, owner, group, null, localName, inodeId,
childrenNum, null);
this.snapshotNumber = snapshotNumber;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 4dcac39..5efede7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -1277,6 +1277,9 @@ public class PBHelper {
if (flag.contains(CreateFlag.OVERWRITE)) {
value |= CreateFlagProto.OVERWRITE.getNumber();
}
+ if (flag.contains(CreateFlag.LAZY_PERSIST)) {
+ value |= CreateFlagProto.LAZY_PERSIST.getNumber();
+ }
return value;
}
@@ -1293,6 +1296,10 @@ public class PBHelper {
== CreateFlagProto.OVERWRITE_VALUE) {
result.add(CreateFlag.OVERWRITE);
}
+ if ((flag & CreateFlagProto.LAZY_PERSIST_VALUE)
+ == CreateFlagProto.LAZY_PERSIST_VALUE) {
+ result.add(CreateFlag.LAZY_PERSIST);
+ }
return new EnumSetWritable<CreateFlag>(result);
}
@@ -1318,6 +1325,7 @@ public class PBHelper {
return new HdfsLocatedFileStatus(
fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
fs.getBlockReplication(), fs.getBlocksize(),
+ fs.hasIsLazyPersist() ? fs.getIsLazyPersist() : false,
fs.getModificationTime(), fs.getAccessTime(),
PBHelper.convert(fs.getPermission()), fs.getOwner(), fs.getGroup(),
fs.getFileType().equals(FileType.IS_SYMLINK) ?
@@ -1366,6 +1374,7 @@ public class PBHelper {
setFileType(fType).
setBlockReplication(fs.getReplication()).
setBlocksize(fs.getBlockSize()).
+ setIsLazyPersist(fs.isLazyPersist()).
setModificationTime(fs.getModificationTime()).
setAccessTime(fs.getAccessTime()).
setPermission(PBHelper.convert(fs.getPermission())).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 54e3181..d03a4e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1438,7 +1438,7 @@ public class FSDirectory implements Closeable {
private HdfsFileStatus getFileInfo4DotSnapshot(String src)
throws UnresolvedLinkException {
if (getINode4DotSnapshot(src) != null) {
- return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ return new HdfsFileStatus(0, true, 0, 0, false, 0, 0, null, null, null, null,
HdfsFileStatus.EMPTY_NAME, -1L, 0, null);
}
return null;
@@ -2300,6 +2300,7 @@ public class FSDirectory implements Closeable {
node.isDirectory(),
replication,
blocksize,
+ false,
node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot),
@@ -2347,7 +2348,7 @@ public class FSDirectory implements Closeable {
HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
- blocksize, node.getModificationTime(snapshot),
+ blocksize, false, node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
getPermissionForFileStatus(node, snapshot),
node.getUserName(snapshot), node.getGroupName(snapshot),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 81d5a22..5ea12b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -106,7 +106,6 @@ import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
import java.security.GeneralSecurityException;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -120,7 +119,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -182,7 +180,6 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -363,7 +360,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
Path symlink = stat.isSymlink() ? new Path(stat.getSymlink()) : null;
Path path = dst != null ? new Path(dst) : new Path(src);
status = new FileStatus(stat.getLen(), stat.isDir(),
- stat.getReplication(), stat.getBlockSize(), stat.getModificationTime(),
+ stat.getReplication(), stat.getBlockSize(), stat.isLazyPersist(),
+ stat.getModificationTime(),
stat.getAccessTime(), stat.getPermission(), stat.getOwner(),
stat.getGroup(), symlink, path);
}
@@ -2445,6 +2443,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+ boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST);
waitForLoadingFSImage();
@@ -2507,8 +2506,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkNameNodeSafeMode("Cannot create file" + src);
src = resolvePath(src, pathComponents);
startFileInternal(pc, src, permissions, holder, clientMachine, create,
- overwrite, createParent, replication, blockSize, suite, edek,
- logRetryCache);
+ overwrite, createParent, replication, blockSize, isLazyPersist,
+ suite, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
FSDirectory.isReservedRawName(srcArg));
} catch (StandbyException se) {
@@ -2548,8 +2547,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
private void startFileInternal(FSPermissionChecker pc, String src,
PermissionStatus permissions, String holder, String clientMachine,
boolean create, boolean overwrite, boolean createParent,
- short replication, long blockSize, CipherSuite suite,
- EncryptedKeyVersion edek, boolean logRetryEntry)
+ short replication, long blockSize, boolean isLazyPersist,
+ CipherSuite suite, EncryptedKeyVersion edek, boolean logRetryEntry)
throws FileAlreadyExistsException, AccessControlException,
UnresolvedLinkException, FileNotFoundException,
ParentNotDirectoryException, RetryStartFileException, IOException {
@@ -2624,7 +2623,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if (parent != null && mkdirsRecursively(parent.toString(),
permissions, true, now())) {
newNode = dir.addFile(src, permissions, replication, blockSize,
- holder, clientMachine);
+ holder, clientMachine);
}
if (newNode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 321630c..98e49e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -244,6 +244,8 @@ public class JsonUtil {
final long aTime = (Long) m.get("accessTime");
final long mTime = (Long) m.get("modificationTime");
final long blockSize = (Long) m.get("blockSize");
+ final boolean isLazyPersist = m.containsKey("lazyPersist")
+ ? (Boolean) m.get("lazyPersist") : false;
final short replication = (short) (long) (Long) m.get("replication");
final long fileId = m.containsKey("fileId") ? (Long) m.get("fileId")
: INodeId.GRANDFATHER_INODE_ID;
@@ -251,9 +253,8 @@ public class JsonUtil {
final int childrenNum = (childrenNumLong == null) ? -1
: childrenNumLong.intValue();
return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
- blockSize, mTime, aTime, permission, owner, group,
- symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum,
- null);
+ blockSize, isLazyPersist, mTime, aTime, permission, owner, group,
+ symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum, null);
}
/** Convert an ExtendedBlock to a Json map. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
index edffc9a..1f30e71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
@@ -64,6 +64,7 @@ enum CreateFlagProto {
CREATE = 0x01; // Create a file
OVERWRITE = 0x02; // Truncate/overwrite a file. Same as POSIX O_TRUNC
APPEND = 0x04; // Append to a file
+ LAZY_PERSIST = 0x10; // File with reduced durability guarantees.
}
message CreateRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index a410224..cbb51f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -263,6 +263,7 @@ message HdfsFileStatusProto {
// Optional field for file encryption
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
+ optional bool isLazyPersist = 16 [default = false];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 74daccc..f0ffb2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -253,12 +253,12 @@ public class TestDFSClientRetries {
anyLong(), any(String[].class))).thenAnswer(answer);
Mockito.doReturn(
- new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
+ new HdfsFileStatus(0, false, 1, 1024, false, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
1010, 0, null)).when(mockNN).getFileInfo(anyString());
Mockito.doReturn(
- new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
+ new HdfsFileStatus(0, false, 1, 1024, false, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
1010, 0, null))
.when(mockNN)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
index 28c253f..f3c0911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -340,12 +340,12 @@ public class TestLease {
}
Mockito.doReturn(
- new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
+ new HdfsFileStatus(0, false, 1, 1024, false, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
1010, 0, null)).when(mcp).getFileInfo(anyString());
Mockito
.doReturn(
- new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
+ new HdfsFileStatus(0, false, 1, 1024, false, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
1010, 0, null))
.when(mcp)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 4cddd60..380f6ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -1017,8 +1017,8 @@ public class TestFsck {
int numChildren = 1;
HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication,
- blockSize, modTime, accessTime, perms, owner, group, symlink, path,
- fileId, numChildren, null);
+ blockSize, false, modTime, accessTime, perms, owner, group, symlink,
+ path, fileId, numChildren, null);
Result res = new Result(conf);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a879d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
index b8150f7..ec82c48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
@@ -62,7 +62,7 @@ public class TestJsonUtil {
final long now = Time.now();
final String parent = "/dir";
final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26,
- now, now + 10, new FsPermission((short) 0644), "user", "group",
+ false, now, now + 10, new FsPermission((short) 0644), "user", "group",
DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"),
INodeId.GRANDFATHER_INODE_ID, 0, null);
final FileStatus fstatus = toFileStatus(status, parent);
[2/2] git commit: HDFS-6924. Add new RAM_DISK storage type. (Arpit
Agarwal)
Posted by ar...@apache.org.
HDFS-6924. Add new RAM_DISK storage type. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fecbf9bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fecbf9bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fecbf9bb
Branch: refs/heads/HDFS-6581
Commit: fecbf9bb6857403a70ee21db7deed4b9cd0aa258
Parents: 20a879d
Author: arp <ar...@apache.org>
Authored: Wed Aug 27 09:03:45 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 09:04:44 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt | 4 ++++
.../src/main/java/org/apache/hadoop/hdfs/StorageType.java | 3 ++-
.../java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 ++++
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto | 2 +-
.../org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java | 9 ++++++---
.../apache/hadoop/hdfs/server/datanode/TestDataDirs.java | 5 ++++-
6 files changed, 21 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index 706c03a..fc6e0e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -2,3 +2,7 @@
HDFS-6921. Add LazyPersist flag to FileStatus. (Arpit Agarwal)
+ HDFS-6924. Add new RAM_DISK storage type. (Arpit Agarwal)
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index 3d8133c..51724f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -32,7 +32,8 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public enum StorageType {
DISK,
- SSD;
+ SSD,
+ RAM_DISK;
public static final StorageType DEFAULT = DISK;
public static final StorageType[] EMPTY_ARRAY = {};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 5efede7..5167597 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -1703,6 +1703,8 @@ public class PBHelper {
return StorageTypeProto.DISK;
case SSD:
return StorageTypeProto.SSD;
+ case RAM_DISK:
+ return StorageTypeProto.RAM_DISK;
default:
throw new IllegalStateException(
"BUG: StorageType not found, type=" + type);
@@ -1731,6 +1733,8 @@ public class PBHelper {
return StorageType.DISK;
case SSD:
return StorageType.SSD;
+ case RAM_DISK:
+ return StorageType.RAM_DISK;
default:
throw new IllegalStateException(
"BUG: StorageTypeProto not found, type=" + type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index cbb51f9..b54638e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -158,6 +158,7 @@ message FsPermissionProto {
enum StorageTypeProto {
DISK = 1;
SSD = 2;
+ RAM_DISK = 3;
}
/**
@@ -260,7 +261,6 @@ message HdfsFileStatusProto {
// Optional field for fileId
optional uint64 fileId = 13 [default = 0]; // default as an invalid id
optional int32 childrenNum = 14 [default = -1];
-
// Optional field for file encryption
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
optional bool isLazyPersist = 16 [default = false];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index cb85c7d..98fd59a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -448,13 +448,16 @@ public class TestPBHelper {
DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2",
AdminStates.DECOMMISSIONED),
DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3",
- AdminStates.NORMAL)
+ AdminStates.NORMAL),
+ DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h4",
+ AdminStates.NORMAL),
};
- String[] storageIDs = {"s1", "s2", "s3"};
+ String[] storageIDs = {"s1", "s2", "s3", "s4"};
StorageType[] media = {
StorageType.DISK,
StorageType.SSD,
- StorageType.DISK
+ StorageType.DISK,
+ StorageType.RAM_DISK
};
LocatedBlock lb = new LocatedBlock(
new ExtendedBlock("bp12", 12345, 10, 53),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fecbf9bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
index 53babb4..c0b4f9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
@@ -44,10 +44,11 @@ public class TestDataDirs {
File dir1 = new File("/dir1");
File dir2 = new File("/dir2");
File dir3 = new File("/dir3");
+ File dir4 = new File("/dir4");
// Verify that a valid string is correctly parsed, and that storage
// type is not case-sensitive
- String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3";
+ String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3,[ram_disk]/dir4";
conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1);
locations = DataNode.getStorageLocations(conf);
assertThat(locations.size(), is(4));
@@ -59,6 +60,8 @@ public class TestDataDirs {
assertThat(locations.get(2).getUri(), is(dir2.toURI()));
assertThat(locations.get(3).getStorageType(), is(StorageType.DISK));
assertThat(locations.get(3).getUri(), is(dir3.toURI()));
+ assertThat(locations.get(4).getStorageType(), is(StorageType.RAM_DISK));
+ assertThat(locations.get(4).getUri(), is(dir4.toURI()));
// Verify that an unrecognized storage type result in an exception.
String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2";