You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/06/19 20:32:14 UTC
svn commit: r1603993 [1/2] - in
/hadoop/common/branches/fs-encryption/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/ hadoop-hdfs/
hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop/fs/
hadoop-hdf...
Author: wang
Date: Thu Jun 19 18:32:13 2014
New Revision: 1603993
URL: http://svn.apache.org/r1603993
Log:
Merge from trunk to branch
Modified:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1603664-1603978
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java Thu Jun 19 18:32:13 2014
@@ -36,6 +36,8 @@ public class NfsConfiguration extends Hd
NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY),
new DeprecationDelta("nfs3.mountd.port",
NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY),
+ new DeprecationDelta("dfs.nfs.exports.cache.size",
+ Nfs3Constant.NFS_EXPORTS_CACHE_SIZE_KEY),
new DeprecationDelta("dfs.nfs.exports.cache.expirytime.millis",
Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY),
new DeprecationDelta("hadoop.nfs.userupdate.milly",
@@ -51,6 +53,16 @@ public class NfsConfiguration extends Hd
new DeprecationDelta("dfs.nfs3.export.point",
NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY),
new DeprecationDelta("nfs.allow.insecure.ports",
- NfsConfigKeys.DFS_NFS_PORT_MONITORING_DISABLED_KEY) });
+ NfsConfigKeys.DFS_NFS_PORT_MONITORING_DISABLED_KEY),
+ new DeprecationDelta("dfs.nfs.keytab.file",
+ NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY),
+ new DeprecationDelta("dfs.nfs.kerberos.principal",
+ NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY),
+ new DeprecationDelta("dfs.nfs.rtmax",
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY),
+ new DeprecationDelta("dfs.nfs.wtmax",
+ NfsConfigKeys.DFS_NFS_MAX_WRITE_TRANSFER_SIZE_KEY),
+ new DeprecationDelta("dfs.nfs.dtmax",
+ NfsConfigKeys.DFS_NFS_MAX_READDIR_TRANSFER_SIZE_KEY) });
}
}
\ No newline at end of file
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Jun 19 18:32:13 2014
@@ -254,9 +254,6 @@ Trunk (Unreleased)
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
- HDFS-6375. Listing extended attributes with the search permission.
- (Charles Lamb via wang)
-
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -454,6 +451,8 @@ Release 2.5.0 - UNRELEASED
HDFS-6530. Fix Balancer documentation. (szetszwo)
+ HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -665,6 +664,12 @@ Release 2.5.0 - UNRELEASED
HDFS-6559. Fix wrong option "dfsadmin -rollingUpgrade start" in the
document. (Akira Ajisaka via Arpit Agarwal)
+ HDFS-6553. Add missing DeprecationDeltas for NFS Kerberos configurations
+ (Stephen Chu via brandonli)
+
+ HDFS-6563. NameNode cannot save fsimage in certain circumstances when
+ snapshots are in use. (atm)
+
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)
@@ -728,6 +733,12 @@ Release 2.5.0 - UNRELEASED
HDFS-6374. setXAttr should require the user to be the owner of the file
or directory (Charles Lamb via wang)
+ HDFS-6375. Listing extended attributes with the search permission.
+ (Charles Lamb via wang)
+
+ HDFS-6492. Support create-time xattrs and atomically setting multiple
+ xattrs. (wang)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1603664-1603978
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java Thu Jun 19 18:32:13 2014
@@ -19,6 +19,8 @@ package org.apache.hadoop.fs;
import java.util.Arrays;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
@@ -105,42 +107,47 @@ public class XAttr {
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((ns == null) ? 0 : ns.hashCode());
- result = prime * result + Arrays.hashCode(value);
- return result;
+ return new HashCodeBuilder(811, 67)
+ .append(name)
+ .append(ns)
+ .append(value)
+ .toHashCode();
}
@Override
public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
+ if (obj == null) { return false; }
+ if (obj == this) { return true; }
+ if (obj.getClass() != getClass()) {
return false;
}
- XAttr other = (XAttr) obj;
- if (name == null) {
- if (other.name != null) {
- return false;
- }
- } else if (!name.equals(other.name)) {
- return false;
- }
- if (ns != other.ns) {
- return false;
- }
- if (!Arrays.equals(value, other.value)) {
+ XAttr rhs = (XAttr) obj;
+ return new EqualsBuilder()
+ .append(ns, rhs.ns)
+ .append(name, rhs.name)
+ .append(value, rhs.value)
+ .isEquals();
+ }
+
+ /**
+ * Similar to {@link #equals(Object)}, except ignores the XAttr value.
+ *
+ * @param obj to compare equality
+ * @return if the XAttrs are equal, ignoring the XAttr value
+ */
+ public boolean equalsIgnoreValue(Object obj) {
+ if (obj == null) { return false; }
+ if (obj == this) { return true; }
+ if (obj.getClass() != getClass()) {
return false;
}
- return true;
+ XAttr rhs = (XAttr) obj;
+ return new EqualsBuilder()
+ .append(ns, rhs.ns)
+ .append(name, rhs.name)
+ .isEquals();
}
-
+
@Override
public String toString() {
return "XAttr [ns=" + ns + ", name=" + name + ", value="
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Thu Jun 19 18:32:13 2014
@@ -2112,6 +2112,9 @@ public class PBHelper {
public static List<XAttrProto> convertXAttrProto(
List<XAttr> xAttrSpec) {
+ if (xAttrSpec == null) {
+ return Lists.newArrayListWithCapacity(0);
+ }
ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
xAttrSpec.size());
for (XAttr a : xAttrSpec) {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java Thu Jun 19 18:32:13 2014
@@ -252,7 +252,7 @@ class Checkpointer extends Daemon {
backupNode.namesystem.writeLock();
try {
- backupNode.namesystem.dir.setReady();
+ backupNode.namesystem.setImageLoaded();
if(backupNode.namesystem.getBlocksTotal() > 0) {
backupNode.namesystem.setBlockTotal();
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Jun 19 18:32:13 2014
@@ -26,11 +26,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
+import java.util.ListIterator;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoCodec;
import org.apache.hadoop.fs.ContentSummary;
@@ -89,15 +89,14 @@ import static org.apache.hadoop.hdfs.pro
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.CRYPTO_XATTR_IV;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.CRYPTO_XATTR_KEY_VERSION_ID;
-/*************************************************
- * FSDirectory stores the filesystem directory state.
- * It handles writing/loading values to disk, and logging
- * changes as we go.
- *
- * It keeps the filename->blockset mapping always-current
- * and logged to disk.
- *
- *************************************************/
+/**
+ * Both FSDirectory and FSNamesystem manage the state of the namespace.
+ * FSDirectory is a pure in-memory data structure, all of whose operations
+ * happen entirely in memory. In contrast, FSNamesystem persists the operations
+ * to the disk.
+ * @see org.apache.hadoop.hdfs.server.namenode.FSNamesystem
+ **/
+@InterfaceAudience.Private
public class FSDirectory implements Closeable {
private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
final INodeDirectory r = new INodeDirectory(
@@ -126,7 +125,6 @@ public class FSDirectory implements Clos
INodeDirectory rootDir;
FSImage fsImage;
private final FSNamesystem namesystem;
- private volatile boolean ready = false;
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
private final int maxComponentLength;
private final int maxDirItems;
@@ -139,7 +137,6 @@ public class FSDirectory implements Clos
// lock to protect the directory and BlockMap
private final ReentrantReadWriteLock dirLock;
- private final Condition cond;
// utility methods to acquire and release read lock and write lock
void readLock() {
@@ -182,7 +179,6 @@ public class FSDirectory implements Clos
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
this.dirLock = new ReentrantReadWriteLock(true); // fair
- this.cond = dirLock.writeLock().newCondition();
rootDir = createRoot(ns);
inodeMap = INodeMap.newInstance(rootDir);
this.fsImage = fsImage;
@@ -240,38 +236,6 @@ public class FSDirectory implements Clos
}
/**
- * Notify that loading of this FSDirectory is complete, and
- * it is ready for use
- */
- void imageLoadComplete() {
- Preconditions.checkState(!ready, "FSDirectory already loaded");
- setReady();
- }
-
- void setReady() {
- if(ready) return;
- writeLock();
- try {
- setReady(true);
- this.nameCache.initialized();
- cond.signalAll();
- } finally {
- writeUnlock();
- }
- }
-
- //This is for testing purposes only
- @VisibleForTesting
- boolean isReady() {
- return ready;
- }
-
- // exposed for unit tests
- protected void setReady(boolean flag) {
- ready = flag;
- }
-
- /**
* Shutdown the filestore
*/
@Override
@@ -279,22 +243,12 @@ public class FSDirectory implements Clos
fsImage.close();
}
- /**
- * Block until the object is ready to be used.
- */
- void waitForReady() {
- if (!ready) {
- writeLock();
- try {
- while (!ready) {
- try {
- cond.await(5000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException ignored) {
- }
- }
- } finally {
- writeUnlock();
- }
+ void markNameCacheInitialized() {
+ writeLock();
+ try {
+ nameCache.initialized();
+ } finally {
+ writeUnlock();
}
}
@@ -320,7 +274,6 @@ public class FSDirectory implements Clos
String clientMachine, DatanodeDescriptor clientNode)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException, AclException {
- waitForReady();
long modTime = now();
INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null,
@@ -350,6 +303,7 @@ public class FSDirectory implements Clos
String path,
PermissionStatus permissions,
List<AclEntry> aclEntries,
+ List<XAttr> xAttrs,
short replication,
long modificationTime,
long atime,
@@ -376,6 +330,10 @@ public class FSDirectory implements Clos
AclStorage.updateINodeAcl(newNode, aclEntries,
Snapshot.CURRENT_STATE_ID);
}
+ if (xAttrs != null) {
+ XAttrStorage.updateINodeXAttrs(newNode, xAttrs,
+ Snapshot.CURRENT_STATE_ID);
+ }
return newNode;
}
} catch (IOException e) {
@@ -393,8 +351,6 @@ public class FSDirectory implements Clos
*/
BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
DatanodeStorageInfo[] targets) throws IOException {
- waitForReady();
-
writeLock();
try {
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
@@ -432,8 +388,6 @@ public class FSDirectory implements Clos
boolean removeBlock(String path, INodeFile fileNode, Block block)
throws IOException {
Preconditions.checkArgument(fileNode.isUnderConstruction());
- waitForReady();
-
writeLock();
try {
return unprotectedRemoveBlock(path, fileNode, block);
@@ -477,7 +431,6 @@ public class FSDirectory implements Clos
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: "
+src+" to "+dst);
}
- waitForReady();
writeLock();
try {
if (!unprotectedRenameTo(src, dst, mtime))
@@ -500,7 +453,6 @@ public class FSDirectory implements Clos
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src
+ " to " + dst);
}
- waitForReady();
writeLock();
try {
if (unprotectedRenameTo(src, dst, mtime, options)) {
@@ -1032,7 +984,6 @@ public class FSDirectory implements Clos
Block[] setReplication(String src, short replication, short[] blockRepls)
throws QuotaExceededException, UnresolvedLinkException,
SnapshotAccessControlException {
- waitForReady();
writeLock();
try {
return unprotectedSetReplication(src, replication, blockRepls);
@@ -1155,7 +1106,6 @@ public class FSDirectory implements Clos
writeLock();
try {
// actual move
- waitForReady();
unprotectedConcat(target, srcs, timestamp);
} finally {
writeUnlock();
@@ -1238,7 +1188,6 @@ public class FSDirectory implements Clos
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
}
- waitForReady();
final long filesRemoved;
writeLock();
try {
@@ -1711,7 +1660,7 @@ public class FSDirectory implements Clos
long nsDelta, long dsDelta, boolean checkQuota)
throws QuotaExceededException {
assert hasWriteLock();
- if (!ready) {
+ if (!namesystem.isImageLoaded()) {
//still initializing. do not check or update quotas.
return;
}
@@ -1904,7 +1853,7 @@ public class FSDirectory implements Clos
*/
private void verifyQuotaForRename(INode[] src, INode[] dst)
throws QuotaExceededException {
- if (!ready || skipQuotaCheck) {
+ if (!namesystem.isImageLoaded() || skipQuotaCheck) {
// Do not check quota if edits log is still being processed
return;
}
@@ -1960,7 +1909,7 @@ public class FSDirectory implements Clos
void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
- if (!ready) {
+ if (!namesystem.isImageLoaded()) {
s += " Please rename it before upgrade.";
}
throw new HadoopIllegalArgumentException(s);
@@ -1987,7 +1936,7 @@ public class FSDirectory implements Clos
getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
final PathComponentTooLongException e = new PathComponentTooLongException(
maxComponentLength, length, p, DFSUtil.bytes2String(childName));
- if (ready) {
+ if (namesystem.isImageLoaded()) {
throw e;
} else {
// Do not throw if edits log is still being processed
@@ -2011,7 +1960,7 @@ public class FSDirectory implements Clos
if (count >= maxDirItems) {
final MaxDirectoryItemsExceededException e
= new MaxDirectoryItemsExceededException(maxDirItems, count);
- if (ready) {
+ if (namesystem.isImageLoaded()) {
e.setPathName(getFullPathName(pathComponents, pos - 1));
throw e;
} else {
@@ -2347,7 +2296,6 @@ public class FSDirectory implements Clos
void reset() {
writeLock();
try {
- setReady(false);
rootDir = createRoot(getFSNamesystem());
inodeMap.clear();
addToInodeMap(rootDir);
@@ -2631,45 +2579,81 @@ public class FSDirectory implements Clos
}
}
- XAttr removeXAttr(String src, XAttr xAttr) throws IOException {
+ /**
+ * Removes a list of XAttrs from an inode at a path.
+ *
+ * @param src path of inode
+ * @param toRemove XAttrs to be removed
+ * @return List of XAttrs that were removed
+ * @throws IOException if the inode does not exist, if quota is exceeded
+ */
+ List<XAttr> removeXAttrs(final String src, final List<XAttr> toRemove)
+ throws IOException {
writeLock();
try {
- return unprotectedRemoveXAttr(src, xAttr);
+ return unprotectedRemoveXAttrs(src, toRemove);
} finally {
writeUnlock();
}
}
-
- XAttr unprotectedRemoveXAttr(String src,
- XAttr xAttr) throws IOException {
+
+ List<XAttr> unprotectedRemoveXAttrs(final String src,
+ final List<XAttr> toRemove) throws IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
- List<XAttr> newXAttrs = filterINodeXAttr(existingXAttrs, xAttr);
+ List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
+ List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
+ removedXAttrs);
if (existingXAttrs.size() != newXAttrs.size()) {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
- return xAttr;
+ return removedXAttrs;
}
return null;
}
-
- List<XAttr> filterINodeXAttr(List<XAttr> existingXAttrs,
- XAttr xAttr) throws QuotaExceededException {
- if (existingXAttrs == null || existingXAttrs.isEmpty()) {
+
+ /**
+ * Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
+ * toFilter and puts them into filtered. Upon completion,
+ * toFilter contains the filter XAttrs that were not found, while
+ * fitleredXAttrs contains the XAttrs that were found.
+ *
+ * @param existingXAttrs Existing XAttrs to be filtered
+ * @param toFilter XAttrs to filter from the existing XAttrs
+ * @param filtered Return parameter, XAttrs that were filtered
+ * @return List of XAttrs that does not contain filtered XAttrs
+ */
+ @VisibleForTesting
+ List<XAttr> filterINodeXAttrs(final List<XAttr> existingXAttrs,
+ final List<XAttr> toFilter, final List<XAttr> filtered) {
+ if (existingXAttrs == null || existingXAttrs.isEmpty() ||
+ toFilter == null || toFilter.isEmpty()) {
return existingXAttrs;
}
-
- List<XAttr> xAttrs = Lists.newArrayListWithCapacity(existingXAttrs.size());
+
+ // Populate a new list with XAttrs that pass the filter
+ List<XAttr> newXAttrs =
+ Lists.newArrayListWithCapacity(existingXAttrs.size());
for (XAttr a : existingXAttrs) {
- if (!(a.getNameSpace() == xAttr.getNameSpace()
- && a.getName().equals(xAttr.getName()))) {
- xAttrs.add(a);
+ boolean add = true;
+ for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
+ ;) {
+ XAttr filter = it.next();
+ if (a.equalsIgnoreValue(filter)) {
+ add = false;
+ it.remove();
+ filtered.add(filter);
+ break;
+ }
+ }
+ if (add) {
+ newXAttrs.add(a);
}
}
-
- return xAttrs;
+
+ return newXAttrs;
}
XAttr createEncryptionZone(String src, String keyId)
@@ -2682,14 +2666,16 @@ public class FSDirectory implements Clos
}
final XAttr keyIdXAttr =
XAttrHelper.buildXAttr(CRYPTO_XATTR_KEY_ID, keyId.getBytes());
- unprotectedSetXAttr(src, keyIdXAttr, EnumSet.of(XAttrSetFlag.CREATE));
+ List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(keyIdXAttr);
+ unprotectedSetXAttrs(src, xattrs, EnumSet.of(XAttrSetFlag.CREATE));
return keyIdXAttr;
} finally {
writeUnlock();
}
}
- XAttr deleteEncryptionZone(String src)
+ List<XAttr> deleteEncryptionZone(String src)
throws IOException {
writeLock();
try {
@@ -2699,71 +2685,107 @@ public class FSDirectory implements Clos
}
final XAttr keyIdXAttr =
XAttrHelper.buildXAttr(CRYPTO_XATTR_KEY_ID, null);
- final XAttr removedXAttr = unprotectedRemoveXAttr(src, keyIdXAttr);
- if (removedXAttr == null) {
+ List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(keyIdXAttr);
+ final List<XAttr> removedXAttrs = unprotectedRemoveXAttrs(src, xattrs);
+ if (removedXAttrs == null || removedXAttrs.isEmpty()) {
throw new IOException(
src + " does not appear to be the root of an encryption zone");
}
- return removedXAttr;
+ return removedXAttrs;
} finally {
writeUnlock();
}
}
- void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
- throws IOException {
+ void setXAttrs(final String src, final List<XAttr> xAttrs,
+ final EnumSet<XAttrSetFlag> flag) throws IOException {
writeLock();
try {
- unprotectedSetXAttr(src, xAttr, flag);
+ unprotectedSetXAttrs(src, xAttrs, flag);
} finally {
writeUnlock();
}
}
- void unprotectedSetXAttr(String src, XAttr xAttr,
- EnumSet<XAttrSetFlag> flag) throws IOException {
+ void unprotectedSetXAttrs(final String src, final List<XAttr> xAttrs,
+ final EnumSet<XAttrSetFlag> flag)
+ throws QuotaExceededException, IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
- List<XAttr> newXAttrs = setINodeXAttr(existingXAttrs, xAttr, flag);
+ List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, xAttrs, flag);
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
}
-
- List<XAttr> setINodeXAttr(List<XAttr> existingXAttrs, XAttr xAttr,
- EnumSet<XAttrSetFlag> flag) throws QuotaExceededException, IOException {
- List<XAttr> xAttrs = Lists.newArrayListWithCapacity(
- existingXAttrs != null ? existingXAttrs.size() + 1 : 1);
+
+ List<XAttr> setINodeXAttrs(final List<XAttr> existingXAttrs,
+ final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ // Check for duplicate XAttrs in toSet
+ // We need to use a custom comparator, so using a HashSet is not suitable
+ for (int i = 0; i < toSet.size(); i++) {
+ for (int j = i + 1; j < toSet.size(); j++) {
+ if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
+ throw new IOException("Cannot specify the same XAttr to be set " +
+ "more than once");
+ }
+ }
+ }
+
+ // Count the current number of user-visible XAttrs for limit checking
int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
- boolean exist = false;
+
+ // The XAttr list is copied to an exactly-sized array when it's stored,
+ // so there's no need to size it precisely here.
+ int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
+ newSize += toSet.size();
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
+
+ // Check if the XAttr already exists to validate with the provided flag
+ for (XAttr xAttr: toSet) {
+ boolean exist = false;
+ if (existingXAttrs != null) {
+ for (XAttr a : existingXAttrs) {
+ if (a.equalsIgnoreValue(xAttr)) {
+ exist = true;
+ break;
+ }
+ }
+ }
+ XAttrSetFlag.validate(xAttr.getName(), exist, flag);
+ // add the new XAttr since it passed validation
+ xAttrs.add(xAttr);
+ if (isUserVisible(xAttr)) {
+ userVisibleXAttrsNum++;
+ }
+ }
+
+ // Add the existing xattrs back in, if they weren't already set
if (existingXAttrs != null) {
- for (XAttr a: existingXAttrs) {
- if ((a.getNameSpace() == xAttr.getNameSpace()
- && a.getName().equals(xAttr.getName()))) {
- exist = true;
- } else {
- xAttrs.add(a);
-
- if (isUserVisible(a)) {
+ for (XAttr existing : existingXAttrs) {
+ boolean alreadySet = false;
+ for (XAttr set : toSet) {
+ if (set.equalsIgnoreValue(existing)) {
+ alreadySet = true;
+ break;
+ }
+ }
+ if (!alreadySet) {
+ xAttrs.add(existing);
+ if (isUserVisible(existing)) {
userVisibleXAttrsNum++;
}
}
}
}
-
- XAttrSetFlag.validate(xAttr.getName(), exist, flag);
- xAttrs.add(xAttr);
-
- if (isUserVisible(xAttr)) {
- userVisibleXAttrsNum++;
- }
-
+
if (userVisibleXAttrsNum > inodeXAttrsLimit) {
throw new IOException("Cannot add additional XAttr to inode, "
+ "would exceed limit of " + inodeXAttrsLimit);
}
-
+
return xAttrs;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Thu Jun 19 18:32:13 2014
@@ -700,12 +700,19 @@ public class FSEditLog implements LogsPu
.setBlocks(newNode.getBlocks())
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
- .setClientMachine(newNode.getFileUnderConstructionFeature().getClientMachine());
+ .setClientMachine(
+ newNode.getFileUnderConstructionFeature().getClientMachine());
AclFeature f = newNode.getAclFeature();
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
+
+ XAttrFeature x = newNode.getXAttrFeature();
+ if (x != null) {
+ op.setXAttrs(x.getXAttrs());
+ }
+
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
@@ -761,6 +768,11 @@ public class FSEditLog implements LogsPu
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
+
+ XAttrFeature x = newNode.getXAttrFeature();
+ if (x != null) {
+ op.setXAttrs(x.getXAttrs());
+ }
logEdit(op);
}
@@ -1054,18 +1066,18 @@ public class FSEditLog implements LogsPu
logEdit(op);
}
- void logSetXAttr(String src, XAttr xAttr, boolean toLogRpcIds) {
+ void logSetXAttrs(String src, List<XAttr> xAttrs, boolean toLogRpcIds) {
final SetXAttrOp op = SetXAttrOp.getInstance();
op.src = src;
- op.xAttr = xAttr;
+ op.xAttrs = xAttrs;
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
- void logRemoveXAttr(String src, XAttr xAttr) {
+ void logRemoveXAttrs(String src, List<XAttr> xAttrs) {
final RemoveXAttrOp op = RemoveXAttrOp.getInstance();
op.src = src;
- op.xAttr = xAttr;
+ op.xAttrs = xAttrs;
logEdit(op);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Thu Jun 19 18:32:13 2014
@@ -355,6 +355,7 @@ public class FSEditLogLoader {
lastInodeId);
newFile = fsDir.unprotectedAddFile(inodeId,
path, addCloseOp.permissions, addCloseOp.aclEntries,
+ addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, true, addCloseOp.clientName,
addCloseOp.clientMachine);
@@ -804,7 +805,7 @@ public class FSEditLogLoader {
}
case OP_SET_XATTR: {
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
- fsDir.unprotectedSetXAttr(setXAttrOp.src, setXAttrOp.xAttr,
+ fsDir.unprotectedSetXAttrs(setXAttrOp.src, setXAttrOp.xAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
@@ -813,7 +814,8 @@ public class FSEditLogLoader {
}
case OP_REMOVE_XATTR: {
RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
- fsDir.unprotectedRemoveXAttr(removeXAttrOp.src, removeXAttrOp.xAttr);
+ fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src,
+ removeXAttrOp.xAttrs);
break;
}
default:
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java Thu Jun 19 18:32:13 2014
@@ -382,6 +382,16 @@ public abstract class FSEditLogOp {
}
}
+ private static List<XAttr> readXAttrsFromEditLog(DataInputStream in,
+ int logVersion) throws IOException {
+ if (!NameNodeLayoutVersion.supports(NameNodeLayoutVersion.Feature.XATTRS,
+ logVersion)) {
+ return null;
+ }
+ XAttrEditLogProto proto = XAttrEditLogProto.parseDelimitedFrom(in);
+ return PBHelper.convertXAttrs(proto.getXAttrsList());
+ }
+
@SuppressWarnings("unchecked")
static abstract class AddCloseOp extends FSEditLogOp implements BlockListUpdatingOp {
int length;
@@ -394,6 +404,7 @@ public abstract class FSEditLogOp {
Block[] blocks;
PermissionStatus permissions;
List<AclEntry> aclEntries;
+ List<XAttr> xAttrs;
String clientName;
String clientMachine;
@@ -461,6 +472,11 @@ public abstract class FSEditLogOp {
return (T)this;
}
+ <T extends AddCloseOp> T setXAttrs(List<XAttr> xAttrs) {
+ this.xAttrs = xAttrs;
+ return (T)this;
+ }
+
<T extends AddCloseOp> T setClientName(String clientName) {
this.clientName = clientName;
return (T)this;
@@ -484,6 +500,9 @@ public abstract class FSEditLogOp {
if (this.opCode == OP_ADD) {
AclEditLogUtil.write(aclEntries, out);
+ XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.build().writeDelimitedTo(out);
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
// write clientId and callId
@@ -546,9 +565,9 @@ public abstract class FSEditLogOp {
this.blocks = readBlocks(in, logVersion);
this.permissions = PermissionStatus.read(in);
- // clientname, clientMachine and block locations of last block.
if (this.opCode == OP_ADD) {
aclEntries = AclEditLogUtil.read(in, logVersion);
+ this.xAttrs = readXAttrsFromEditLog(in, logVersion);
this.clientName = FSImageSerialization.readString(in);
this.clientMachine = FSImageSerialization.readString(in);
// read clientId and callId
@@ -1343,6 +1362,7 @@ public abstract class FSEditLogOp {
long timestamp;
PermissionStatus permissions;
List<AclEntry> aclEntries;
+ List<XAttr> xAttrs;
private MkdirOp() {
super(OP_MKDIR);
@@ -1377,6 +1397,11 @@ public abstract class FSEditLogOp {
return this;
}
+ MkdirOp setXAttrs(List<XAttr> xAttrs) {
+ this.xAttrs = xAttrs;
+ return this;
+ }
+
@Override
public
void writeFields(DataOutputStream out) throws IOException {
@@ -1386,6 +1411,9 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeLong(timestamp, out); // atime, unused at this
permissions.write(out);
AclEditLogUtil.write(aclEntries, out);
+ XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.build().writeDelimitedTo(out);
}
@Override
@@ -1430,6 +1458,8 @@ public abstract class FSEditLogOp {
this.permissions = PermissionStatus.read(in);
aclEntries = AclEditLogUtil.read(in, logVersion);
+
+ xAttrs = readXAttrsFromEditLog(in, logVersion);
}
@Override
@@ -1451,6 +1481,8 @@ public abstract class FSEditLogOp {
builder.append(opCode);
builder.append(", txid=");
builder.append(txid);
+ builder.append(", xAttrs=");
+ builder.append(xAttrs);
builder.append("]");
return builder.toString();
}
@@ -1468,6 +1500,9 @@ public abstract class FSEditLogOp {
if (aclEntries != null) {
appendAclEntriesToXml(contentHandler, aclEntries);
}
+ if (xAttrs != null) {
+ appendXAttrsToXml(contentHandler, xAttrs);
+ }
}
@Override void fromXml(Stanza st) throws InvalidXmlException {
@@ -1477,6 +1512,7 @@ public abstract class FSEditLogOp {
this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
this.permissions = permissionStatusFromXml(st);
aclEntries = readAclEntriesFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
}
}
@@ -3499,7 +3535,7 @@ public abstract class FSEditLogOp {
}
static class RemoveXAttrOp extends FSEditLogOp {
- XAttr xAttr;
+ List<XAttr> xAttrs;
String src;
private RemoveXAttrOp() {
@@ -3514,7 +3550,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttr = PBHelper.convertXAttr(p.getXAttr());
+ xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
}
@Override
@@ -3523,25 +3559,25 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
- b.setXAttr(PBHelper.convertXAttrProto(xAttr));
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
- appendXAttrToXml(contentHandler, xAttr);
+ appendXAttrsToXml(contentHandler, xAttrs);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
- xAttr = readXAttrFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
}
}
static class SetXAttrOp extends FSEditLogOp {
- XAttr xAttr;
+ List<XAttr> xAttrs;
String src;
private SetXAttrOp() {
@@ -3556,7 +3592,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttr = PBHelper.convertXAttr(p.getXAttr());
+ xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
readRpcIds(in, logVersion);
}
@@ -3566,7 +3602,7 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
- b.setXAttr(PBHelper.convertXAttrProto(xAttr));
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
// clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
@@ -3575,14 +3611,14 @@ public abstract class FSEditLogOp {
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
- appendXAttrToXml(contentHandler, xAttr);
+ appendXAttrsToXml(contentHandler, xAttrs);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
- xAttr = readXAttrFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
readRpcIdsFromXml(st);
}
}
@@ -4202,42 +4238,48 @@ public abstract class FSEditLogOp {
}
return aclEntries;
}
-
- private static void appendXAttrToXml(ContentHandler contentHandler,
- XAttr xAttr) throws SAXException {
- contentHandler.startElement("", "", "XATTR", new AttributesImpl());
- XMLUtils.addSaxString(contentHandler, "NAMESPACE",
- xAttr.getNameSpace().toString());
- XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName());
- if (xAttr.getValue() != null) {
- try {
- XMLUtils.addSaxString(contentHandler, "VALUE",
- XAttrCodec.encodeValue(xAttr.getValue(), XAttrCodec.HEX));
- } catch (IOException e) {
- throw new SAXException(e);
+
+ private static void appendXAttrsToXml(ContentHandler contentHandler,
+ List<XAttr> xAttrs) throws SAXException {
+ for (XAttr xAttr: xAttrs) {
+ contentHandler.startElement("", "", "XATTR", new AttributesImpl());
+ XMLUtils.addSaxString(contentHandler, "NAMESPACE",
+ xAttr.getNameSpace().toString());
+ XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName());
+ if (xAttr.getValue() != null) {
+ try {
+ XMLUtils.addSaxString(contentHandler, "VALUE",
+ XAttrCodec.encodeValue(xAttr.getValue(), XAttrCodec.HEX));
+ } catch (IOException e) {
+ throw new SAXException(e);
+ }
}
+ contentHandler.endElement("", "", "XATTR");
}
- contentHandler.endElement("", "", "XATTR");
}
-
- private static XAttr readXAttrFromXml(Stanza st)
+
+ private static List<XAttr> readXAttrsFromXml(Stanza st)
throws InvalidXmlException {
if (!st.hasChildren("XATTR")) {
return null;
}
-
- Stanza a = st.getChildren("XATTR").get(0);
- XAttr.Builder builder = new XAttr.Builder();
- builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
- setName(a.getValue("NAME"));
- String v = a.getValueOrNull("VALUE");
- if (v != null) {
- try {
- builder.setValue(XAttrCodec.decodeValue(v));
- } catch (IOException e) {
- throw new InvalidXmlException(e.toString());
+
+ List<Stanza> stanzas = st.getChildren("XATTR");
+ List<XAttr> xattrs = Lists.newArrayListWithCapacity(stanzas.size());
+ for (Stanza a: stanzas) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
+ setName(a.getValue("NAME"));
+ String v = a.getValueOrNull("VALUE");
+ if (v != null) {
+ try {
+ builder.setValue(XAttrCodec.decodeValue(v));
+ } catch (IOException e) {
+ throw new InvalidXmlException(e.toString());
+ }
}
+ xattrs.add(builder.build());
}
- return builder.build();
+ return xattrs;
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java Thu Jun 19 18:32:13 2014
@@ -533,8 +533,10 @@ public final class FSImageFormatPBINode
INodeSection.INodeFile.Builder b = buildINodeFile(n,
parent.getSaverContext());
- for (Block block : n.getBlocks()) {
- b.addBlocks(PBHelper.convert(block));
+ if (n.getBlocks() != null) {
+ for (Block block : n.getBlocks()) {
+ b.addBlocks(PBHelper.convert(block));
+ }
}
FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Jun 19 18:32:13 2014
@@ -108,6 +108,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -530,6 +531,59 @@ public class FSNamesystem implements Nam
private final Map<String, EncryptionZone> encryptionZones;
+ private volatile boolean imageLoaded = false;
+ private final Condition cond;
+ /**
+ * Notify that loading of this FSDirectory is complete, and
+ * it is imageLoaded for use
+ */
+ void imageLoadComplete() {
+ Preconditions.checkState(!imageLoaded, "FSDirectory already loaded");
+ setImageLoaded();
+ }
+
+ void setImageLoaded() {
+ if(imageLoaded) return;
+ writeLock();
+ try {
+ setImageLoaded(true);
+ dir.markNameCacheInitialized();
+ cond.signalAll();
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ //This is for testing purposes only
+ @VisibleForTesting
+ boolean isImageLoaded() {
+ return imageLoaded;
+ }
+
+ // exposed for unit tests
+ protected void setImageLoaded(boolean flag) {
+ imageLoaded = flag;
+ }
+
+ /**
+ * Block until the object is imageLoaded to be used.
+ */
+ void waitForLoadingFSImage() {
+ if (!imageLoaded) {
+ writeLock();
+ try {
+ while (!imageLoaded) {
+ try {
+ cond.await(5000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+ }
+
/**
* Set the last allocated inode id when fsimage or editlog is loaded.
*/
@@ -571,6 +625,7 @@ public class FSNamesystem implements Nam
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
+ setImageLoaded(false);
}
@VisibleForTesting
@@ -700,6 +755,7 @@ public class FSNamesystem implements Nam
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
LOG.info("fsLock is fair:" + fair);
fsLock = new FSNamesystemLock(fair);
+ cond = fsLock.writeLock().newCondition();
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -976,7 +1032,7 @@ public class FSNamesystem implements Nam
}
writeUnlock();
}
- dir.imageLoadComplete();
+ imageLoadComplete();
}
private void startSecretManager() {
@@ -1895,6 +1951,7 @@ public class FSNamesystem implements Nam
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2170,6 +2227,7 @@ public class FSNamesystem implements Nam
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2297,6 +2355,8 @@ public class FSNamesystem implements Nam
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2785,6 +2845,7 @@ public class FSNamesystem implements Nam
Block newBlock = null;
long offset;
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3007,6 +3068,7 @@ public class FSNamesystem implements Nam
}
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3105,6 +3167,7 @@ public class FSNamesystem implements Nam
boolean success = false;
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3304,6 +3367,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename " + src);
+ waitForLoadingFSImage();
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
checkOperation(OperationCategory.WRITE);
@@ -3411,6 +3475,7 @@ public class FSNamesystem implements Nam
false);
}
+ waitForLoadingFSImage();
long mtime = now();
dir.renameTo(src, dst, mtime, options);
getEditLog().logRename(src, dst, mtime, logRetryCache, options);
@@ -3484,6 +3549,8 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean ret = false;
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3957,6 +4024,8 @@ public class FSNamesystem implements Nam
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4158,6 +4227,7 @@ public class FSNamesystem implements Nam
INodeFile pendingFile, int latestSnapshot) throws IOException,
UnresolvedLinkException {
assert hasWriteLock();
+
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), src);
@@ -4169,6 +4239,7 @@ public class FSNamesystem implements Nam
// since we just remove the uc feature from pendingFile
final INodeFile newFile = pendingFile.toCompleteFile(now());
+ waitForLoadingFSImage();
// close file and persist block allocations for this file
closeFile(src, newFile);
@@ -4227,6 +4298,7 @@ public class FSNamesystem implements Nam
+ ")");
checkOperation(OperationCategory.WRITE);
String src = "";
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4572,7 +4644,7 @@ public class FSNamesystem implements Nam
*/
private void closeFile(String path, INodeFile file) {
assert hasWriteLock();
- dir.waitForReady();
+ waitForLoadingFSImage();
// file is closed
getEditLog().logCloseFile(path, file);
if (NameNode.stateChangeLog.isDebugEnabled()) {
@@ -4596,7 +4668,7 @@ public class FSNamesystem implements Nam
boolean createParent, boolean logRetryCache)
throws UnresolvedLinkException, FileAlreadyExistsException,
QuotaExceededException, SnapshotAccessControlException, AclException {
- dir.waitForReady();
+ waitForLoadingFSImage();
final long modTime = now();
if (createParent) {
@@ -5859,7 +5931,7 @@ public class FSNamesystem implements Nam
boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (!pc.isSuperUser()) {
- dir.waitForReady();
+ waitForLoadingFSImage();
readLock();
try {
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
@@ -6326,6 +6398,7 @@ public class FSNamesystem implements Nam
+ ", newNodes=" + Arrays.asList(newNodes)
+ ", clientName=" + clientName
+ ")");
+ waitForLoadingFSImage();
writeLock();
boolean success = false;
try {
@@ -8206,7 +8279,9 @@ public class FSNamesystem implements Nam
}
final XAttr keyIdXAttr = dir.createEncryptionZone(src, keyId);
- getEditLog().logSetXAttr(src, keyIdXAttr, logRetryCache);
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(keyIdXAttr);
+ getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
encryptionZones.put(src, new EncryptionZone(src, keyId));
resultingStat = getAuditFileInfo(src, false);
} finally {
@@ -8283,9 +8358,9 @@ public class FSNamesystem implements Nam
throw new IOException("Directory " + src +
" is not the root of an encryption zone.");
}
- final XAttr removedXAttr = dir.deleteEncryptionZone(src);
- if (removedXAttr != null) {
- getEditLog().logRemoveXAttr(src, removedXAttr);
+ final List<XAttr> removedXAttrs = dir.deleteEncryptionZone(src);
+ if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+ getEditLog().logRemoveXAttrs(src, removedXAttrs);
}
encryptionZones.remove(src);
resultingStat = getAuditFileInfo(src, false);
@@ -8379,8 +8454,10 @@ public class FSNamesystem implements Nam
checkOwner(pc, src);
checkPathAccess(pc, src, FsAction.WRITE);
}
- dir.setXAttr(src, xAttr, flag);
- getEditLog().logSetXAttr(src, xAttr, logRetryCache);
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ dir.setXAttrs(src, xAttrs, flag);
+ getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -8500,10 +8577,12 @@ public class FSNamesystem implements Nam
checkOwner(pc, src);
checkPathAccess(pc, src, FsAction.WRITE);
}
-
- XAttr removedXAttr = dir.removeXAttr(src, xAttr);
- if (removedXAttr != null) {
- getEditLog().logRemoveXAttr(src, removedXAttr);
+
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
+ if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+ getEditLog().logRemoveXAttrs(src, removedXAttrs);
}
resultingStat = getAuditFileInfo(src, false);
} catch (AccessControlException e) {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Thu Jun 19 18:32:13 2014
@@ -1064,7 +1064,7 @@ public class SecondaryNameNode implement
} finally {
dstNamesystem.writeUnlock();
}
- dstNamesystem.dir.imageLoadComplete();
+ dstNamesystem.imageLoadComplete();
}
// error simulation code for junit test
CheckpointFaultInjector.getInstance().duringMerge();
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java Thu Jun 19 18:32:13 2014
@@ -159,7 +159,7 @@ public class FileWithSnapshotFeature imp
// resize the array.
final BlockInfo[] newBlocks;
if (n == 0) {
- newBlocks = null;
+ newBlocks = BlockInfo.EMPTY_ARRAY;
} else {
newBlocks = new BlockInfo[n];
System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto Thu Jun 19 18:32:13 2014
@@ -35,8 +35,8 @@ message XAttrProto {
}
message XAttrEditLogProto {
- required string src = 1;
- optional XAttrProto xAttr = 2;
+ optional string src = 1;
+ repeated XAttrProto xAttrs = 2;
}
enum XAttrSetFlagProto {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Thu Jun 19 18:32:13 2014
@@ -32,7 +32,6 @@ import java.io.IOException;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Mockito.*;
/**
@@ -50,6 +49,7 @@ public class TestCommitBlockSynchronizat
final DatanodeStorageInfo[] targets = {};
FSNamesystem namesystem = new FSNamesystem(conf, image);
+ namesystem.setImageLoaded(true);
FSNamesystem namesystemSpy = spy(namesystem);
BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java Thu Jun 19 18:32:13 2014
@@ -24,7 +24,9 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.EnumSet;
import java.util.List;
+import java.util.Random;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -36,7 +38,6 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -45,6 +46,11 @@ import org.junit.Test;
import com.google.common.collect.Lists;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
/**
* Test {@link FSDirectory}, the in-memory namespace tree.
*/
@@ -74,6 +80,10 @@ public class TestFSDirectory {
private DistributedFileSystem hdfs;
+ private static final int numGeneratedXAttrs = 256;
+ private static final ImmutableList<XAttr> generatedXAttrs =
+ ImmutableList.copyOf(generateXAttrs(numGeneratedXAttrs));
+
@Before
public void setUp() throws Exception {
conf = new Configuration();
@@ -119,25 +129,16 @@ public class TestFSDirectory {
for(; (line = in.readLine()) != null; ) {
line = line.trim();
if (!line.isEmpty() && !line.contains("snapshot")) {
- Assert.assertTrue("line=" + line,
+ assertTrue("line=" + line,
line.startsWith(INodeDirectory.DUMPTREE_LAST_ITEM)
- || line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM));
+ || line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM)
+ );
checkClassName(line);
}
}
}
@Test
- public void testReset() throws Exception {
- fsdir.reset();
- Assert.assertFalse(fsdir.isReady());
- final INodeDirectory root = (INodeDirectory) fsdir.getINode("/");
- Assert.assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
- fsdir.imageLoadComplete();
- Assert.assertTrue(fsdir.isReady());
- }
-
- @Test
public void testSkipQuotaCheck() throws Exception {
try {
// set quota. nsQuota of 1 means no files can be created
@@ -176,7 +177,7 @@ public class TestFSDirectory {
int i = line.lastIndexOf('(');
int j = line.lastIndexOf('@');
final String classname = line.substring(i+1, j);
- Assert.assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
+ assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
|| classname.startsWith(INodeDirectory.class.getSimpleName()));
}
@@ -193,22 +194,185 @@ public class TestFSDirectory {
// Adding a system namespace xAttr, isn't affected by inode xAttrs limit.
XAttr newXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM).
setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build();
- List<XAttr> xAttrs = fsdir.setINodeXAttr(existingXAttrs, newXAttr,
+ List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(1);
+ newXAttrs.add(newXAttr);
+ List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
- Assert.assertEquals(xAttrs.size(), 3);
+ assertEquals(xAttrs.size(), 3);
// Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace(
XAttr.NameSpace.TRUSTED).setName("a4").
setValue(new byte[]{0x34, 0x34, 0x34}).build();
+ newXAttrs.set(0, newXAttr1);
try {
- fsdir.setINodeXAttr(existingXAttrs, newXAttr1,
+ fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
- Assert.fail("Setting user visable xattr on inode should fail if " +
+ fail("Setting user visible xattr on inode should fail if " +
"reaching limit.");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot add additional XAttr " +
"to inode, would exceed limit", e);
}
}
+
+ /**
+ * Verify that the first <i>num</i> generatedXAttrs are present in
+ * newXAttrs.
+ */
+ private static void verifyXAttrsPresent(List<XAttr> newXAttrs,
+ final int num) {
+ assertEquals("Unexpected number of XAttrs after multiset", num,
+ newXAttrs.size());
+ for (int i=0; i<num; i++) {
+ XAttr search = generatedXAttrs.get(i);
+ assertTrue("Did not find set XAttr " + search + " + after multiset",
+ newXAttrs.contains(search));
+ }
+ }
+
+ private static List<XAttr> generateXAttrs(final int numXAttrs) {
+ List<XAttr> generatedXAttrs = Lists.newArrayListWithCapacity(numXAttrs);
+ for (int i=0; i<numXAttrs; i++) {
+ XAttr xAttr = (new XAttr.Builder())
+ .setNameSpace(XAttr.NameSpace.SYSTEM)
+ .setName("a" + i)
+ .setValue(new byte[] { (byte) i, (byte) (i + 1), (byte) (i + 2) })
+ .build();
+ generatedXAttrs.add(xAttr);
+ }
+ return generatedXAttrs;
+ }
+
+ /**
+ * Test setting and removing multiple xattrs via single operations
+ */
+ @Test(timeout=300000)
+ public void testXAttrMultiSetRemove() throws Exception {
+ List<XAttr> existingXAttrs = Lists.newArrayListWithCapacity(0);
+
+ // Keep adding a random number of xattrs and verifying until exhausted
+ final Random rand = new Random(0xFEEDA);
+ int numExpectedXAttrs = 0;
+ while (numExpectedXAttrs < numGeneratedXAttrs) {
+ LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
+ final int numToAdd = rand.nextInt(5)+1;
+
+ List<XAttr> toAdd = Lists.newArrayListWithCapacity(numToAdd);
+ for (int i = 0; i < numToAdd; i++) {
+ if (numExpectedXAttrs >= numGeneratedXAttrs) {
+ break;
+ }
+ toAdd.add(generatedXAttrs.get(numExpectedXAttrs));
+ numExpectedXAttrs++;
+ }
+ LOG.info("Attempting to add " + toAdd.size() + " XAttrs");
+ for (int i = 0; i < toAdd.size(); i++) {
+ LOG.info("Will add XAttr " + toAdd.get(i));
+ }
+ List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE));
+ verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
+ existingXAttrs = newXAttrs;
+ }
+
+ // Keep removing a random number of xattrs and verifying until all gone
+ while (numExpectedXAttrs > 0) {
+ LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
+ final int numToRemove = rand.nextInt(5)+1;
+ List<XAttr> toRemove = Lists.newArrayListWithCapacity(numToRemove);
+ for (int i = 0; i < numToRemove; i++) {
+ if (numExpectedXAttrs == 0) {
+ break;
+ }
+ toRemove.add(generatedXAttrs.get(numExpectedXAttrs-1));
+ numExpectedXAttrs--;
+ }
+ final int expectedNumToRemove = toRemove.size();
+ LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
+ List<XAttr> removedXAttrs = Lists.newArrayList();
+ List<XAttr> newXAttrs = fsdir.filterINodeXAttrs(existingXAttrs,
+ toRemove, removedXAttrs);
+ assertEquals("Unexpected number of removed XAttrs",
+ expectedNumToRemove, removedXAttrs.size());
+ verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
+ existingXAttrs = newXAttrs;
+ }
+ }
+
+ @Test(timeout=300000)
+ public void testXAttrMultiAddRemoveErrors() throws Exception {
+
+ // Test that the same XAttr can not be multiset twice
+ List<XAttr> existingXAttrs = Lists.newArrayList();
+ List<XAttr> toAdd = Lists.newArrayList();
+ toAdd.add(generatedXAttrs.get(0));
+ toAdd.add(generatedXAttrs.get(1));
+ toAdd.add(generatedXAttrs.get(2));
+ toAdd.add(generatedXAttrs.get(0));
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .CREATE));
+ fail("Specified the same xattr to be set twice");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Cannot specify the same " +
+ "XAttr to be set", e);
+ }
+
+ // Test that CREATE and REPLACE flags are obeyed
+ toAdd.remove(generatedXAttrs.get(0));
+ existingXAttrs.add(generatedXAttrs.get(0));
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .CREATE));
+ fail("Set XAttr that is already set without REPLACE flag");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("already exists", e);
+ }
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .REPLACE));
+ fail("Set XAttr that does not exist without the CREATE flag");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("does not exist", e);
+ }
+
+ // Sanity test for CREATE
+ toAdd.remove(generatedXAttrs.get(0));
+ List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE));
+ assertEquals("Unexpected toAdd size", 2, toAdd.size());
+ for (XAttr x : toAdd) {
+ assertTrue("Did not find added XAttr " + x, newXAttrs.contains(x));
+ }
+ existingXAttrs = newXAttrs;
+
+ // Sanity test for REPLACE
+ toAdd = Lists.newArrayList();
+ for (int i=0; i<3; i++) {
+ XAttr xAttr = (new XAttr.Builder())
+ .setNameSpace(XAttr.NameSpace.SYSTEM)
+ .setName("a" + i)
+ .setValue(new byte[] { (byte) (i*2) })
+ .build();
+ toAdd.add(xAttr);
+ }
+ newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.REPLACE));
+ assertEquals("Unexpected number of new XAttrs", 3, newXAttrs.size());
+ for (int i=0; i<3; i++) {
+ assertArrayEquals("Unexpected XAttr value",
+ new byte[] {(byte)(i*2)}, newXAttrs.get(i).getValue());
+ }
+ existingXAttrs = newXAttrs;
+
+ // Sanity test for CREATE+REPLACE
+ toAdd = Lists.newArrayList();
+ for (int i=0; i<4; i++) {
+ toAdd.add(generatedXAttrs.get(i));
+ }
+ newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ verifyXAttrsPresent(newXAttrs, 4);
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Thu Jun 19 18:32:13 2014
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
@@ -194,4 +195,22 @@ public class TestFSNamesystem {
assertFalse(rwLock.isWriteLockedByCurrentThread());
assertEquals(0, rwLock.getWriteHoldCount());
}
+
+ @Test
+ public void testReset() throws Exception {
+ Configuration conf = new Configuration();
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+ FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+ fsn.imageLoadComplete();
+ assertTrue(fsn.isImageLoaded());
+ fsn.clear();
+ assertFalse(fsn.isImageLoaded());
+ final INodeDirectory root = (INodeDirectory) fsn.getFSDirectory()
+ .getINode("/");
+ assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
+ fsn.imageLoadComplete();
+ assertTrue(fsn.isImageLoaded());
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java Thu Jun 19 18:32:13 2014
@@ -19,12 +19,9 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
-import static org.apache.hadoop.util.Time.now;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
@@ -57,7 +54,7 @@ public class TestFsLimits {
FSEditLog editLog = mock(FSEditLog.class);
doReturn(editLog).when(fsImage).getEditLog();
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
- fsn.getFSDirectory().setReady(fsIsReady);
+ fsn.setImageLoaded(fsIsReady);
return fsn;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java Thu Jun 19 18:32:13 2014
@@ -28,12 +28,14 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -396,4 +398,39 @@ public class TestSnapshotBlocksMap {
assertEquals(1, blks.length);
assertEquals(BLOCKSIZE, blks[0].getNumBytes());
}
+
+ /**
+ * Make sure that a delete of a non-zero-length file which results in a
+ * zero-length file in a snapshot works.
+ */
+ @Test
+ public void testDeletionOfLaterBlocksWithZeroSizeFirstBlock() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final byte[] testData = "foo bar baz".getBytes();
+
+ // Create a zero-length file.
+ DFSTestUtil.createFile(hdfs, bar, 0, REPLICATION, 0L);
+ assertEquals(0, fsdir.getINode4Write(bar.toString()).asFile().getBlocks().length);
+
+ // Create a snapshot that includes that file.
+ SnapshotTestHelper.createSnapshot(hdfs, foo, "s0");
+
+ // Extend that file.
+ FSDataOutputStream out = hdfs.append(bar);
+ out.write(testData);
+ out.close();
+ INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
+ BlockInfo[] blks = barNode.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(testData.length, blks[0].getNumBytes());
+
+ // Delete the file.
+ hdfs.delete(bar, true);
+
+ // Now make sure that the NN can still save an fsimage successfully.
+ cluster.getNameNode().getRpcServer().setSafeMode(
+ SafeModeAction.SAFEMODE_ENTER, false);
+ cluster.getNameNode().getRpcServer().saveNamespace();
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java Thu Jun 19 18:32:13 2014
@@ -249,10 +249,10 @@ public class TestXAttrWithSnapshot {
private static void doSnapshotRootRemovalAssertions(Path path,
Path snapshotPath) throws Exception {
Map<String, byte[]> xattrs = hdfs.getXAttrs(path);
- Assert.assertEquals(xattrs.size(), 0);
+ Assert.assertEquals(0, xattrs.size());
xattrs = hdfs.getXAttrs(snapshotPath);
- Assert.assertEquals(xattrs.size(), 2);
+ Assert.assertEquals(2, xattrs.size());
Assert.assertArrayEquals(value1, xattrs.get(name1));
Assert.assertArrayEquals(value2, xattrs.get(name2));
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1603993&r1=1603992&r2=1603993&view=diff
==============================================================================
Binary files - no diff available.