You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:25 UTC
svn commit: r1619012 [18/35] - in
/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Aug 19 23:49:39 2014
@@ -62,6 +62,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY;
@@ -83,14 +85,13 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SUPPORT_APPEND_KEY;
import static org.apache.hadoop.util.Time.now;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -115,6 +116,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -141,7 +143,10 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -154,6 +159,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -175,8 +181,8 @@ import org.apache.hadoop.hdfs.protocol.Q
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@@ -213,8 +219,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -226,6 +230,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
@@ -446,7 +451,6 @@ public class FSNamesystem implements Nam
NameNodeResourceChecker nnResourceChecker;
private final FsServerDefaults serverDefaults;
- private final boolean supportAppends;
private final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
private volatile SafeModeInfo safeMode; // safe mode information
@@ -520,7 +524,65 @@ public class FSNamesystem implements Nam
private final RetryCache retryCache;
- private final AclConfigFlag aclConfigFlag;
+ private final NNConf nnConf;
+
+ private volatile boolean imageLoaded = false;
+ private final Condition cond;
+
+ private final FSImage fsImage;
+
+ private boolean randomizeBlockLocationsPerBlock;
+
+ /**
+ * Notify that loading of this FSDirectory is complete, and
+ * it is imageLoaded for use
+ */
+ void imageLoadComplete() {
+ Preconditions.checkState(!imageLoaded, "FSDirectory already loaded");
+ setImageLoaded();
+ }
+
+ void setImageLoaded() {
+ if(imageLoaded) return;
+ writeLock();
+ try {
+ setImageLoaded(true);
+ dir.markNameCacheInitialized();
+ cond.signalAll();
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ //This is for testing purposes only
+ @VisibleForTesting
+ boolean isImageLoaded() {
+ return imageLoaded;
+ }
+
+ // exposed for unit tests
+ protected void setImageLoaded(boolean flag) {
+ imageLoaded = flag;
+ }
+
+ /**
+ * Block until the object is imageLoaded to be used.
+ */
+ void waitForLoadingFSImage() {
+ if (!imageLoaded) {
+ writeLock();
+ try {
+ while (!imageLoaded) {
+ try {
+ cond.await(5000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+ }
/**
* Set the last allocated inode id when fsimage or editlog is loaded.
@@ -563,6 +625,7 @@ public class FSNamesystem implements Nam
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
+ setImageLoaded(false);
}
@VisibleForTesting
@@ -690,6 +753,8 @@ public class FSNamesystem implements Nam
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
LOG.info("fsLock is fair:" + fair);
fsLock = new FSNamesystemLock(fair);
+ cond = fsLock.writeLock().newCondition();
+ this.fsImage = fsImage;
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -754,8 +819,6 @@ public class FSNamesystem implements Nam
DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT);
this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);
- this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT);
- LOG.info("Append Enabled: " + supportAppends);
this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
@@ -779,9 +842,13 @@ public class FSNamesystem implements Nam
alwaysUseDelegationTokensForTests = conf.getBoolean(
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
+
+ this.randomizeBlockLocationsPerBlock = conf.getBoolean(
+ DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK,
+ DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT);
this.dtSecretManager = createDelegationTokenSecretManager(conf);
- this.dir = new FSDirectory(fsImage, this, conf);
+ this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
this.cacheManager = new CacheManager(this, conf, blockManager);
this.safeMode = new SafeModeInfo(conf);
@@ -789,7 +856,7 @@ public class FSNamesystem implements Nam
this.isDefaultAuditLogger = auditLoggers.size() == 1 &&
auditLoggers.get(0) instanceof DefaultAuditLogger;
this.retryCache = ignoreRetryCache ? null : initRetryCache(conf);
- this.aclConfigFlag = new AclConfigFlag(conf);
+ this.nnConf = new NNConf(conf);
} catch(IOException e) {
LOG.error(getClass().getSimpleName() + " initialization failed.", e);
close();
@@ -921,7 +988,8 @@ public class FSNamesystem implements Nam
}
// This will start a new log segment and write to the seen_txid file, so
// we shouldn't do it when coming up in standby state
- if (!haEnabled || (haEnabled && startOpt == StartupOption.UPGRADE)) {
+ if (!haEnabled || (haEnabled && startOpt == StartupOption.UPGRADE)
+ || (haEnabled && startOpt == StartupOption.UPGRADEONLY)) {
fsImage.openEditLogForWrite();
}
success = true;
@@ -931,7 +999,7 @@ public class FSNamesystem implements Nam
}
writeUnlock();
}
- dir.imageLoadComplete();
+ imageLoadComplete();
}
private void startSecretManager() {
@@ -1009,7 +1077,7 @@ public class FSNamesystem implements Nam
LOG.info("Starting services required for active state");
writeLock();
try {
- FSEditLog editLog = dir.fsImage.getEditLog();
+ FSEditLog editLog = getFSImage().getEditLog();
if (!editLog.isOpenForWrite()) {
// During startup, we're already open for write during initialization.
@@ -1038,12 +1106,12 @@ public class FSNamesystem implements Nam
metaSaveAsString());
}
- long nextTxId = dir.fsImage.getLastAppliedTxId() + 1;
+ long nextTxId = getFSImage().getLastAppliedTxId() + 1;
LOG.info("Will take over writing edit logs at txnid " +
nextTxId);
editLog.setNextTxId(nextTxId);
- dir.fsImage.editLog.openForWrite();
+ getFSImage().editLog.openForWrite();
}
// Enable quota checks.
@@ -1109,9 +1177,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
stopSecretManager();
- if (leaseManager != null) {
- leaseManager.stopMonitor();
- }
+ leaseManager.stopMonitor();
if (nnrmthread != null) {
((NameNodeResourceMonitor) nnrmthread.getRunnable()).stopMonitor();
nnrmthread.interrupt();
@@ -1120,16 +1186,18 @@ public class FSNamesystem implements Nam
((NameNodeEditLogRoller)nnEditLogRoller.getRunnable()).stop();
nnEditLogRoller.interrupt();
}
- if (dir != null && dir.fsImage != null) {
- if (dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null) {
+ if (getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
// Update the fsimage with the last txid that we wrote
// so that the tailer starts from the right spot.
- dir.fsImage.updateLastAppliedTxIdFromWritten();
+ getFSImage().updateLastAppliedTxIdFromWritten();
+ }
+ if (cacheManager != null) {
+ cacheManager.stopMonitorThread();
+ cacheManager.clearDirectiveStats();
}
- cacheManager.stopMonitorThread();
- cacheManager.clearDirectiveStats();
blockManager.getDatanodeManager().clearPendingCachingCommands();
blockManager.getDatanodeManager().setShouldSendCachingCommands(false);
// Don't want to keep replication queues when not in Active.
@@ -1147,9 +1215,9 @@ public class FSNamesystem implements Nam
*/
void startStandbyServices(final Configuration conf) throws IOException {
LOG.info("Starting services required for standby state");
- if (!dir.fsImage.editLog.isOpenForRead()) {
+ if (!getFSImage().editLog.isOpenForRead()) {
// During startup, we're already open for read.
- dir.fsImage.editLog.initSharedJournalsForRead();
+ getFSImage().editLog.initSharedJournalsForRead();
}
blockManager.setPostponeBlocksFromFuture(true);
@@ -1196,8 +1264,8 @@ public class FSNamesystem implements Nam
if (editLogTailer != null) {
editLogTailer.stop();
}
- if (dir != null && dir.fsImage != null && dir.fsImage.editLog != null) {
- dir.fsImage.editLog.close();
+ if (dir != null && getFSImage() != null && getFSImage().editLog != null) {
+ getFSImage().editLog.close();
}
}
@@ -1353,7 +1421,7 @@ public class FSNamesystem implements Nam
/**
* Returns edit directories that are shared between primary and secondary.
* @param conf configuration
- * @return Collection of edit directories.
+ * @return collection of edit directories from {@code conf}
*/
public static List<URI> getSharedEditsDirs(Configuration conf) {
// don't use getStorageDirs here, because we want an empty default
@@ -1440,9 +1508,9 @@ public class FSNamesystem implements Nam
* Version of @see #getNamespaceInfo() that is not protected by a lock.
*/
NamespaceInfo unprotectedGetNamespaceInfo() {
- return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(),
+ return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(),
getClusterId(), getBlockPoolId(),
- dir.fsImage.getStorage().getCTime());
+ getFSImage().getStorage().getCTime());
}
/**
@@ -1460,12 +1528,10 @@ public class FSNamesystem implements Nam
try {
stopActiveServices();
stopStandbyServices();
- if (dir != null) {
- dir.close();
- }
} catch (IOException ie) {
- LOG.error("Error closing FSDirectory", ie);
+ } finally {
IOUtils.cleanup(LOG, dir);
+ IOUtils.cleanup(LOG, fsImage);
}
}
}
@@ -1578,6 +1644,7 @@ public class FSNamesystem implements Nam
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
dir.setPermission(src, permission);
+ getEditLog().logSetPermissions(src, permission);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -1623,6 +1690,7 @@ public class FSNamesystem implements Nam
}
}
dir.setOwner(src, username, group);
+ getEditLog().logSetOwner(src, username, group);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -1641,15 +1709,17 @@ public class FSNamesystem implements Nam
LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true,
true);
if (blocks != null) {
- blockManager.getDatanodeManager().sortLocatedBlocks(
- clientMachine, blocks.getLocatedBlocks());
-
+ blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
+ blocks.getLocatedBlocks(), randomizeBlockLocationsPerBlock);
+
+ // lastBlock is not part of getLocatedBlocks(), might need to sort it too
LocatedBlock lastBlock = blocks.getLastLocatedBlock();
if (lastBlock != null) {
- ArrayList<LocatedBlock> lastBlockList = new ArrayList<LocatedBlock>();
+ ArrayList<LocatedBlock> lastBlockList =
+ Lists.newArrayListWithCapacity(1);
lastBlockList.add(lastBlock);
- blockManager.getDatanodeManager().sortLocatedBlocks(
- clientMachine, lastBlockList);
+ blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
+ lastBlockList, randomizeBlockLocationsPerBlock);
}
}
return blocks;
@@ -1751,7 +1821,11 @@ public class FSNamesystem implements Nam
if (isReadOp) {
continue;
}
- dir.setTimes(src, inode, -1, now, false, iip.getLatestSnapshotId());
+ boolean changed = dir.setTimes(inode, -1, now, false,
+ iip.getLatestSnapshotId());
+ if (changed) {
+ getEditLog().logTimes(src, -1, now);
+ }
}
}
final long fileSize = iip.isSnapshot() ?
@@ -1789,9 +1863,9 @@ public class FSNamesystem implements Nam
* before we start actual move.
*
* This does not support ".inodes" relative path
- * @param target target file path to concatenate into
- * @param srcs files that are concatenated
- * @throws IOException
+ * @param target target to concat into
+ * @param srcs file that will be concatenated
+ * @throws IOException on error
*/
void concat(String target, String [] srcs)
throws IOException, UnresolvedLinkException {
@@ -1842,6 +1916,7 @@ public class FSNamesystem implements Nam
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -1962,7 +2037,9 @@ public class FSNamesystem implements Nam
Arrays.toString(srcs) + " to " + target);
}
- dir.concat(target,srcs, logRetryCache);
+ long timestamp = now();
+ dir.concat(target, srcs, timestamp);
+ getEditLog().logConcat(target, srcs, timestamp, logRetryCache);
}
/**
@@ -2003,7 +2080,11 @@ public class FSNamesystem implements Nam
final INodesInPath iip = dir.getINodesInPath4Write(src);
final INode inode = iip.getLastINode();
if (inode != null) {
- dir.setTimes(src, inode, mtime, atime, true, iip.getLatestSnapshotId());
+ boolean changed = dir.setTimes(inode, mtime, atime, true,
+ iip.getLatestSnapshotId());
+ if (changed) {
+ getEditLog().logTimes(src, mtime, atime);
+ }
resultingStat = getAuditFileInfo(src, false);
} else {
throw new FileNotFoundException("File/Directory " + src + " does not exist.");
@@ -2072,7 +2153,7 @@ public class FSNamesystem implements Nam
checkFsObjectLimit();
// add symbolic link to namespace
- dir.addSymlink(link, target, dirPerms, createParent, logRetryCache);
+ addSymlink(link, target, dirPerms, createParent, logRetryCache);
resultingStat = getAuditFileInfo(link, false);
} finally {
writeUnlock();
@@ -2111,6 +2192,7 @@ public class FSNamesystem implements Nam
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2124,6 +2206,7 @@ public class FSNamesystem implements Nam
final Block[] blocks = dir.setReplication(src, replication, blockRepls);
isFile = blocks != null;
if (isFile) {
+ getEditLog().logSetReplication(src, replication);
blockManager.setReplication(blockRepls[0], blockRepls[1], src, blocks);
}
} finally {
@@ -2237,6 +2320,8 @@ public class FSNamesystem implements Nam
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2321,11 +2406,16 @@ public class FSNamesystem implements Nam
}
checkFsObjectLimit();
- final DatanodeDescriptor clientNode =
- blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
+ INodeFile newNode = null;
+
+ // Always do an implicit mkdirs for parent directory tree.
+ Path parent = new Path(src).getParent();
+ if (parent != null && mkdirsRecursively(parent.toString(),
+ permissions, true, now())) {
+ newNode = dir.addFile(src, permissions, replication, blockSize,
+ holder, clientMachine);
+ }
- INodeFile newNode = dir.addFile(src, permissions, replication, blockSize,
- holder, clientMachine, clientNode);
if (newNode == null) {
throw new IOException("Unable to add " + src + " to namespace");
}
@@ -2391,11 +2481,15 @@ public class FSNamesystem implements Nam
// finalizeINodeFileUnderConstruction so we need to refresh
// the referenced file.
myFile = INodeFile.valueOf(dir.getINode(src), src, true);
-
- final DatanodeDescriptor clientNode =
- blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
- return prepareFileForWrite(src, myFile, holder, clientMachine, clientNode,
- true, iip.getLatestSnapshotId(), logRetryCache);
+ final BlockInfo lastBlock = myFile.getLastBlock();
+ // Check that the block has at least minimum replication.
+ if(lastBlock != null && lastBlock.isComplete() &&
+ !getBlockManager().isSufficientlyReplicated(lastBlock)) {
+ throw new IOException("append: lastBlock=" + lastBlock +
+ " of src=" + src + " is not sufficiently replicated yet.");
+ }
+ return prepareFileForWrite(src, myFile, holder, clientMachine, true,
+ iip.getLatestSnapshotId(), logRetryCache);
} catch (IOException ie) {
NameNode.stateChangeLog.warn("DIR* NameSystem.append: " +ie.getMessage());
throw ie;
@@ -2410,7 +2504,6 @@ public class FSNamesystem implements Nam
* @param file existing file object
* @param leaseHolder identifier of the lease holder on this file
* @param clientMachine identifier of the client machine
- * @param clientNode if the client is collocated with a DN, that DN's descriptor
* @param writeToEditLog whether to persist this change to the edit log
* @param logRetryCache whether to record RPC ids in editlog for retry cache
* rebuilding
@@ -2419,17 +2512,23 @@ public class FSNamesystem implements Nam
* @throws IOException
*/
LocatedBlock prepareFileForWrite(String src, INodeFile file,
- String leaseHolder, String clientMachine, DatanodeDescriptor clientNode,
- boolean writeToEditLog, int latestSnapshot, boolean logRetryCache)
+ String leaseHolder, String clientMachine,
+ boolean writeToEditLog,
+ int latestSnapshot, boolean logRetryCache)
throws IOException {
- file = file.recordModification(latestSnapshot);
- final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine,
- clientNode);
+ file.recordModification(latestSnapshot);
+ final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine);
leaseManager.addLease(cons.getFileUnderConstructionFeature()
.getClientName(), src);
LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
+ if (ret != null) {
+ // update the quota: use the preferred block size for UC block
+ final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
+ dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication());
+ }
+
if (writeToEditLog) {
getEditLog().logOpenFile(src, cons, logRetryCache);
}
@@ -2499,10 +2598,10 @@ public class FSNamesystem implements Nam
// We found the lease for this file. And surprisingly the original
// holder is trying to recreate this file. This should never occur.
//
+
if (!force && lease != null) {
Lease leaseFile = leaseManager.getLeaseByPath(src);
- if ((leaseFile != null && leaseFile.equals(lease)) ||
- lease.getHolder().equals(holder)) {
+ if (leaseFile != null && leaseFile.equals(lease)) {
throw new AlreadyBeingCreatedException(
"failed to create file " + src + " for " + holder +
" for client " + clientMachine +
@@ -2599,12 +2698,6 @@ public class FSNamesystem implements Nam
+ ", clientMachine=" + clientMachine);
}
boolean skipSync = false;
- if (!supportAppends) {
- throw new UnsupportedOperationException(
- "Append is not enabled on this NameNode. Use the " +
- DFS_SUPPORT_APPEND_KEY + " configuration option to enable it.");
- }
-
LocatedBlock lb = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
@@ -2681,9 +2774,10 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- final INode[] inodes = analyzeFileState(
- src, fileId, clientName, previous, onRetryBlock).getINodes();
- final INodeFile pendingFile = inodes[inodes.length - 1].asFile();
+ FileState fileState = analyzeFileState(
+ src, fileId, clientName, previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
// This is a retry. Just return the last block if having locations.
@@ -2696,7 +2790,8 @@ public class FSNamesystem implements Nam
+ maxBlocksPerFile);
}
blockSize = pendingFile.getPreferredBlockSize();
- clientNode = pendingFile.getFileUnderConstructionFeature().getClientNode();
+ clientNode = blockManager.getDatanodeManager().getDatanodeByHost(
+ pendingFile.getFileUnderConstructionFeature().getClientMachine());
replication = pendingFile.getFileReplication();
} finally {
readUnlock();
@@ -2711,16 +2806,17 @@ public class FSNamesystem implements Nam
Block newBlock = null;
long offset;
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
// Run the full analysis again, since things could have changed
// while chooseTarget() was executing.
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- INodesInPath inodesInPath =
+ FileState fileState =
analyzeFileState(src, fileId, clientName, previous, onRetryBlock);
- INode[] inodes = inodesInPath.getINodes();
- final INodeFile pendingFile = inodes[inodes.length - 1].asFile();
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
if (onRetryBlock[0] != null) {
if (onRetryBlock[0].getLocations().length > 0) {
@@ -2742,9 +2838,10 @@ public class FSNamesystem implements Nam
// allocate new block, record block locations in INode.
newBlock = createNewBlock();
+ INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
saveAllocatedBlock(src, inodesInPath, newBlock, targets);
- dir.persistNewBlock(src, pendingFile);
+ persistNewBlock(src, pendingFile);
offset = pendingFile.computeFileSize();
} finally {
writeUnlock();
@@ -2755,7 +2852,17 @@ public class FSNamesystem implements Nam
return makeLocatedBlock(newBlock, targets, offset);
}
- INodesInPath analyzeFileState(String src,
+ static class FileState {
+ public final INodeFile inode;
+ public final String path;
+
+ public FileState(INodeFile inode, String fullPath) {
+ this.inode = inode;
+ this.path = fullPath;
+ }
+ }
+
+ FileState analyzeFileState(String src,
long fileId,
String clientName,
ExtendedBlock previous,
@@ -2772,9 +2879,20 @@ public class FSNamesystem implements Nam
checkFsObjectLimit();
Block previousBlock = ExtendedBlock.getLocalBlock(previous);
- final INodesInPath iip = dir.getINodesInPath4Write(src);
- final INodeFile pendingFile
- = checkLease(src, fileId, clientName, iip.getLastINode());
+ INode inode;
+ if (fileId == INodeId.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ final INodesInPath iip = dir.getINodesInPath4Write(src);
+ inode = iip.getLastINode();
+ } else {
+ // Newer clients pass the inode ID, so we can just get the inode
+ // directly.
+ inode = dir.getInode(fileId);
+ if (inode != null) src = inode.getFullPathName();
+ }
+ final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
BlockInfo lastBlockInFile = pendingFile.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
@@ -2832,7 +2950,7 @@ public class FSNamesystem implements Nam
onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
offset);
- return iip;
+ return new FileState(pendingFile, src);
} else {
// Case 3
throw new IOException("Cannot allocate block in " + src + ": " +
@@ -2845,7 +2963,7 @@ public class FSNamesystem implements Nam
if (!checkFileProgress(pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
- return iip;
+ return new FileState(pendingFile, src);
}
LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
@@ -2858,8 +2976,9 @@ public class FSNamesystem implements Nam
}
/** @see ClientProtocol#getAdditionalDatanode */
- LocatedBlock getAdditionalDatanode(String src, final ExtendedBlock blk,
- final DatanodeInfo[] existings, final String[] storageIDs,
+ LocatedBlock getAdditionalDatanode(String src, long fileId,
+ final ExtendedBlock blk, final DatanodeInfo[] existings,
+ final String[] storageIDs,
final Set<Node> excludes,
final int numAdditionalNodes, final String clientName
) throws IOException {
@@ -2879,8 +2998,20 @@ public class FSNamesystem implements Nam
src = FSDirectory.resolvePath(src, pathComponents, dir);
//check lease
- final INodeFile file = checkLease(src, clientName);
- clientnode = file.getFileUnderConstructionFeature().getClientNode();
+ final INode inode;
+ if (fileId == INodeId.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ inode = dir.getINode(src);
+ } else {
+ inode = dir.getInode(fileId);
+ if (inode != null) src = inode.getFullPathName();
+ }
+ final INodeFile file = checkLease(src, clientName, inode, fileId);
+ String clientMachine = file.getFileUnderConstructionFeature()
+ .getClientMachine();
+ clientnode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
preferredblocksize = file.getPreferredBlockSize();
//find datanode storages
@@ -2903,7 +3034,7 @@ public class FSNamesystem implements Nam
/**
* The client would like to let go of the given block
*/
- boolean abandonBlock(ExtendedBlock b, String src, String holder)
+ boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
throws LeaseExpiredException, FileNotFoundException,
UnresolvedLinkException, IOException {
if(NameNode.stateChangeLog.isDebugEnabled()) {
@@ -2912,16 +3043,28 @@ public class FSNamesystem implements Nam
}
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- checkNameNodeSafeMode("Cannot abandon block " + b + " for fle" + src);
+ checkNameNodeSafeMode("Cannot abandon block " + b + " for file" + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
+ final INode inode;
+ if (fileId == INodeId.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ inode = dir.getINode(src);
+ } else {
+ inode = dir.getInode(fileId);
+ if (inode != null) src = inode.getFullPathName();
+ }
+ final INodeFile file = checkLease(src, holder, inode, fileId);
+
//
// Remove the block from the pending creates list
//
- INodeFile file = checkLease(src, holder);
boolean removed = dir.removeBlock(src, file,
ExtendedBlock.getLocalBlock(b));
if (!removed) {
@@ -2931,7 +3074,7 @@ public class FSNamesystem implements Nam
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
+ b + " is removed from pendingCreates");
}
- dir.persistBlocks(src, file, false);
+ persistBlocks(src, file, false);
} finally {
writeUnlock();
}
@@ -2939,39 +3082,45 @@ public class FSNamesystem implements Nam
return true;
}
-
- /** make sure that we still have the lease on this file. */
- private INodeFile checkLease(String src, String holder)
- throws LeaseExpiredException, UnresolvedLinkException,
- FileNotFoundException {
- return checkLease(src, INodeId.GRANDFATHER_INODE_ID, holder,
- dir.getINode(src));
- }
-
- private INodeFile checkLease(String src, long fileId, String holder,
- INode inode) throws LeaseExpiredException, FileNotFoundException {
+
+ private INodeFile checkLease(String src, String holder, INode inode,
+ long fileId)
+ throws LeaseExpiredException, FileNotFoundException {
assert hasReadLock();
- if (inode == null || !inode.isFile()) {
+ final String ident = src + " (inode " + fileId + ")";
+ if (inode == null) {
Lease lease = leaseManager.getLease(holder);
throw new LeaseExpiredException(
- "No lease on " + src + ": File does not exist. "
+ "No lease on " + ident + ": File does not exist. "
+ (lease != null ? lease.toString()
: "Holder " + holder + " does not have any open files."));
}
+ if (!inode.isFile()) {
+ Lease lease = leaseManager.getLease(holder);
+ throw new LeaseExpiredException(
+ "No lease on " + ident + ": INode is not a regular file. "
+ + (lease != null ? lease.toString()
+ : "Holder " + holder + " does not have any open files."));
+ }
final INodeFile file = inode.asFile();
if (!file.isUnderConstruction()) {
Lease lease = leaseManager.getLease(holder);
throw new LeaseExpiredException(
- "No lease on " + src + ": File is not open for writing. "
+ "No lease on " + ident + ": File is not open for writing. "
+ (lease != null ? lease.toString()
: "Holder " + holder + " does not have any open files."));
}
+ // No further modification is allowed on a deleted file.
+ // A file is considered deleted, if it is not in the inodeMap or is marked
+ // as deleted in the snapshot feature.
+ if (isFileDeleted(file)) {
+ throw new FileNotFoundException(src);
+ }
String clientName = file.getFileUnderConstructionFeature().getClientName();
if (holder != null && !clientName.equals(holder)) {
- throw new LeaseExpiredException("Lease mismatch on " + src + " owned by "
- + clientName + " but is accessed by " + holder);
+ throw new LeaseExpiredException("Lease mismatch on " + ident +
+ " owned by " + clientName + " but is accessed by " + holder);
}
- INodeId.checkId(fileId, file);
return file;
}
@@ -2992,6 +3141,7 @@ public class FSNamesystem implements Nam
boolean success = false;
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3014,10 +3164,20 @@ public class FSNamesystem implements Nam
String holder, Block last, long fileId) throws SafeModeException,
UnresolvedLinkException, IOException {
assert hasWriteLock();
- final INodesInPath iip = dir.getLastINodeInPath(src);
final INodeFile pendingFile;
try {
- pendingFile = checkLease(src, fileId, holder, iip.getINode(0));
+ final INode inode;
+ if (fileId == INodeId.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ final INodesInPath iip = dir.getLastINodeInPath(src);
+ inode = iip.getINode(0);
+ } else {
+ inode = dir.getInode(fileId);
+ if (inode != null) src = inode.getFullPathName();
+ }
+ pendingFile = checkLease(src, holder, inode, fileId);
} catch (LeaseExpiredException lee) {
final INode inode = dir.getINode(src);
if (inode != null
@@ -3032,9 +3192,9 @@ public class FSNamesystem implements Nam
final Block realLastBlock = inode.asFile().getLastBlock();
if (Block.matchingIdAndGenStamp(last, realLastBlock)) {
NameNode.stateChangeLog.info("DIR* completeFile: " +
- "request from " + holder + " to complete " + src +
- " which is already closed. But, it appears to be an RPC " +
- "retry. Returning success");
+ "request from " + holder + " to complete inode " + fileId +
+ "(" + src + ") which is already closed. But, it appears to be " +
+ "an RPC retry. Returning success");
return true;
}
}
@@ -3054,7 +3214,7 @@ public class FSNamesystem implements Nam
}
finalizeINodeFileUnderConstruction(src, pendingFile,
- iip.getLatestSnapshotId());
+ Snapshot.CURRENT_STATE_ID);
return true;
}
@@ -3181,6 +3341,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename " + src);
+ waitForLoadingFSImage();
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
checkOperation(OperationCategory.WRITE);
@@ -3214,13 +3375,16 @@ public class FSNamesystem implements Nam
// Rename does not operates on link targets
// Do not resolveLink when checking permissions of src and dst
// Check write access to parent of src
- checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false);
+ checkPermission(pc, src, false, null, FsAction.WRITE, null, null,
+ false, false);
// Check write access to ancestor of dst
checkPermission(pc, actualdst, false, FsAction.WRITE, null, null, null,
- false);
+ false, false);
}
- if (dir.renameTo(src, dst, logRetryCache)) {
+ long mtime = now();
+ if (dir.renameTo(src, dst, mtime)) {
+ getEditLog().logRename(src, dst, mtime, logRetryCache);
return true;
}
return false;
@@ -3278,12 +3442,17 @@ public class FSNamesystem implements Nam
// Rename does not operates on link targets
// Do not resolveLink when checking permissions of src and dst
// Check write access to parent of src
- checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false);
+ checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false,
+ false);
// Check write access to ancestor of dst
- checkPermission(pc, dst, false, FsAction.WRITE, null, null, null, false);
+ checkPermission(pc, dst, false, FsAction.WRITE, null, null, null, false,
+ false);
}
- dir.renameTo(src, dst, logRetryCache, options);
+ waitForLoadingFSImage();
+ long mtime = now();
+ dir.renameTo(src, dst, mtime, options);
+ getEditLog().logRename(src, dst, mtime, logRetryCache, options);
}
/**
@@ -3354,22 +3523,31 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean ret = false;
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot delete " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (!recursive && dir.isNonEmptyDirectory(src)) {
- throw new IOException(src + " is non empty");
+ throw new PathIsNotEmptyDirectoryException(src + " is non empty");
}
if (enforcePermission && isPermissionEnabled) {
checkPermission(pc, src, false, null, FsAction.WRITE, null,
- FsAction.ALL, false);
+ FsAction.ALL, true, false);
}
+ long mtime = now();
// Unlink the target directory from directory tree
- if (!dir.delete(src, collectedBlocks, removedINodes, logRetryCache)) {
+ long filesRemoved = dir.delete(src, collectedBlocks, removedINodes,
+ mtime);
+ if (filesRemoved < 0) {
return false;
}
+ getEditLog().logDelete(src, mtime, logRetryCache);
+ incrDeletedFileCount(filesRemoved);
+ // Blocks/INodes will be handled later
+ removePathAndBlocks(src, null, removedINodes, true);
ret = true;
} finally {
writeUnlock();
@@ -3377,13 +3555,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
removeBlocks(collectedBlocks); // Incremental deletion of blocks
collectedBlocks.clear();
- dir.writeLock();
- try {
- dir.removeFromInodeMap(removedINodes);
- } finally {
- dir.writeUnlock();
- }
- removedINodes.clear();
+
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
+ src +" is removed");
@@ -3421,14 +3593,24 @@ public class FSNamesystem implements Nam
* @param blocks Containing the list of blocks to be deleted from blocksMap
* @param removedINodes Containing the list of inodes to be removed from
* inodesMap
+ * @param acquireINodeMapLock Whether to acquire the lock for inode removal
*/
void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks,
- List<INode> removedINodes) {
+ List<INode> removedINodes, final boolean acquireINodeMapLock) {
assert hasWriteLock();
leaseManager.removeLeaseWithPrefixPath(src);
// remove inodes from inodesMap
if (removedINodes != null) {
- dir.removeFromInodeMap(removedINodes);
+ if (acquireINodeMapLock) {
+ dir.writeLock();
+ }
+ try {
+ dir.removeFromInodeMap(removedINodes);
+ } finally {
+ if (acquireINodeMapLock) {
+ dir.writeUnlock();
+ }
+ }
removedINodes.clear();
}
if (blocks == null) {
@@ -3516,7 +3698,8 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (isPermissionEnabled) {
- checkPermission(pc, src, false, null, null, null, null, resolveLink);
+ checkPermission(pc, src, false, null, null, null, null, false,
+ resolveLink);
}
stat = dir.getFileInfo(src, resolveLink);
} catch (AccessControlException e) {
@@ -3537,8 +3720,10 @@ public class FSNamesystem implements Nam
StandbyException, IOException {
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.READ);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
readLock();
try {
+ src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOperation(OperationCategory.READ);
if (isPermissionEnabled) {
checkTraverse(pc, src);
@@ -3589,7 +3774,7 @@ public class FSNamesystem implements Nam
src = FSDirectory.resolvePath(src, pathComponents, dir);
status = mkdirsInternal(pc, src, permissions, createParent);
if (status) {
- resultingStat = dir.getFileInfo(src, false);
+ resultingStat = getAuditFileInfo(src, false);
}
} finally {
writeUnlock();
@@ -3628,13 +3813,119 @@ public class FSNamesystem implements Nam
// create multiple inodes.
checkFsObjectLimit();
- if (!dir.mkdirs(src, permissions, false, now())) {
+ if (!mkdirsRecursively(src, permissions, false, now())) {
throw new IOException("Failed to create directory: " + src);
}
return true;
}
/**
+ * Create a directory
+ * If ancestor directories do not exist, automatically create them.
+
+ * @param src string representation of the path to the directory
+ * @param permissions the permission of the directory
+ * @param inheritPermission if the permission of the directory should inherit
+ * from its parent or not. u+wx is implicitly added to
+ * the automatically created directories, and to the
+ * given directory if inheritPermission is true
+ * @param now creation time
+ * @return true if the operation succeeds false otherwise
+ * @throws QuotaExceededException if directory creation violates
+ * any quota limit
+ * @throws UnresolvedLinkException if a symlink is encountered in src.
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ */
+ private boolean mkdirsRecursively(String src, PermissionStatus permissions,
+ boolean inheritPermission, long now)
+ throws FileAlreadyExistsException, QuotaExceededException,
+ UnresolvedLinkException, SnapshotAccessControlException,
+ AclException {
+ src = FSDirectory.normalizePath(src);
+ String[] names = INode.getPathNames(src);
+ byte[][] components = INode.getPathComponents(names);
+ final int lastInodeIndex = components.length - 1;
+
+ dir.writeLock();
+ try {
+ INodesInPath iip = dir.getExistingPathINodes(components);
+ if (iip.isSnapshot()) {
+ throw new SnapshotAccessControlException(
+ "Modification on RO snapshot is disallowed");
+ }
+ INode[] inodes = iip.getINodes();
+
+ // find the index of the first null in inodes[]
+ StringBuilder pathbuilder = new StringBuilder();
+ int i = 1;
+ for(; i < inodes.length && inodes[i] != null; i++) {
+ pathbuilder.append(Path.SEPARATOR).append(names[i]);
+ if (!inodes[i].isDirectory()) {
+ throw new FileAlreadyExistsException(
+ "Parent path is not a directory: "
+ + pathbuilder + " "+inodes[i].getLocalName());
+ }
+ }
+
+ // default to creating parent dirs with the given perms
+ PermissionStatus parentPermissions = permissions;
+
+ // if not inheriting and it's the last inode, there's no use in
+ // computing perms that won't be used
+ if (inheritPermission || (i < lastInodeIndex)) {
+ // if inheriting (ie. creating a file or symlink), use the parent dir,
+ // else the supplied permissions
+ // NOTE: the permissions of the auto-created directories violate posix
+ FsPermission parentFsPerm = inheritPermission
+ ? inodes[i-1].getFsPermission() : permissions.getPermission();
+
+ // ensure that the permissions allow user write+execute
+ if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) {
+ parentFsPerm = new FsPermission(
+ parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE),
+ parentFsPerm.getGroupAction(),
+ parentFsPerm.getOtherAction()
+ );
+ }
+
+ if (!parentPermissions.getPermission().equals(parentFsPerm)) {
+ parentPermissions = new PermissionStatus(
+ parentPermissions.getUserName(),
+ parentPermissions.getGroupName(),
+ parentFsPerm
+ );
+ // when inheriting, use same perms for entire path
+ if (inheritPermission) permissions = parentPermissions;
+ }
+ }
+
+ // create directories beginning from the first null index
+ for(; i < inodes.length; i++) {
+ pathbuilder.append(Path.SEPARATOR).append(names[i]);
+ dir.unprotectedMkdir(allocateNewInodeId(), iip, i, components[i],
+ (i < lastInodeIndex) ? parentPermissions : permissions, null,
+ now);
+ if (inodes[i] == null) {
+ return false;
+ }
+ // Directory creation also count towards FilesCreated
+ // to match count of FilesDeleted metric.
+ NameNode.getNameNodeMetrics().incrFilesCreated();
+
+ final String cur = pathbuilder.toString();
+ getEditLog().logMkDir(cur, inodes[i]);
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug(
+ "mkdirs: created directory " + cur);
+ }
+ }
+ } finally {
+ dir.writeUnlock();
+ }
+ return true;
+ }
+
+ /**
* Get the content summary for a specific file/dir.
*
* @param src The string representation of the path to the file
@@ -3678,7 +3969,7 @@ public class FSNamesystem implements Nam
*
* Note: This does not support ".inodes" relative path.
*/
- void setQuota(String path, long nsQuota, long dsQuota)
+ void setQuota(String path, long nsQuota, long dsQuota)
throws IOException, UnresolvedLinkException {
checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
@@ -3686,7 +3977,12 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set quota on " + path);
- dir.setQuota(path, nsQuota, dsQuota);
+ INodeDirectory changed = dir.setQuota(path, nsQuota, dsQuota);
+ if (changed != null) {
+ final Quota.Counts q = changed.getQuotaCounts();
+ getEditLog().logSetQuota(path,
+ q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE));
+ }
} finally {
writeUnlock();
}
@@ -3695,27 +3991,41 @@ public class FSNamesystem implements Nam
/** Persist all metadata about this file.
* @param src The string representation of the path
+ * @param fileId The inode ID that we're fsyncing. Older clients will pass
+ * INodeId.GRANDFATHER_INODE_ID here.
* @param clientName The string representation of the client
* @param lastBlockLength The length of the last block
* under construction reported from client.
* @throws IOException if path does not exist
*/
- void fsync(String src, String clientName, long lastBlockLength)
+ void fsync(String src, long fileId, String clientName, long lastBlockLength)
throws IOException, UnresolvedLinkException {
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot fsync file " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
- INodeFile pendingFile = checkLease(src, clientName);
+ final INode inode;
+ if (fileId == INodeId.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ inode = dir.getINode(src);
+ } else {
+ inode = dir.getInode(fileId);
+ if (inode != null) src = inode.getFullPathName();
+ }
+ final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
if (lastBlockLength > 0) {
pendingFile.getFileUnderConstructionFeature().updateLengthOfLastBlock(
pendingFile, lastBlockLength);
}
- dir.persistBlocks(src, pendingFile, false);
+ persistBlocks(src, pendingFile, false);
} finally {
writeUnlock();
}
@@ -3787,20 +4097,10 @@ public class FSNamesystem implements Nam
final BlockInfo lastBlock = pendingFile.getLastBlock();
BlockUCState lastBlockState = lastBlock.getBlockUCState();
BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
- boolean penultimateBlockMinReplication;
- BlockUCState penultimateBlockState;
- if (penultimateBlock == null) {
- penultimateBlockState = BlockUCState.COMPLETE;
- // If penultimate block doesn't exist then its minReplication is met
- penultimateBlockMinReplication = true;
- } else {
- penultimateBlockState = BlockUCState.COMMITTED;
- penultimateBlockMinReplication =
+
+ // If penultimate block doesn't exist then its minReplication is met
+ boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
blockManager.checkMinReplication(penultimateBlock);
- }
- assert penultimateBlockState == BlockUCState.COMPLETE ||
- penultimateBlockState == BlockUCState.COMMITTED :
- "Unexpected state of penultimate block in " + src;
switch(lastBlockState) {
case COMPLETE:
@@ -3906,19 +4206,21 @@ public class FSNamesystem implements Nam
INodeFile pendingFile, int latestSnapshot) throws IOException,
UnresolvedLinkException {
assert hasWriteLock();
+
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), src);
- pendingFile = pendingFile.recordModification(latestSnapshot);
+ pendingFile.recordModification(latestSnapshot);
// The file is no longer pending.
// Create permanent INode, update blocks. No need to replace the inode here
// since we just remove the uc feature from pendingFile
final INodeFile newFile = pendingFile.toCompleteFile(now());
+ waitForLoadingFSImage();
// close file and persist block allocations for this file
- dir.closeFile(src, newFile);
+ closeFile(src, newFile);
blockManager.checkReplication(newFile);
}
@@ -3975,6 +4277,7 @@ public class FSNamesystem implements Nam
+ ")");
checkOperation(OperationCategory.WRITE);
String src = "";
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3997,7 +4300,30 @@ public class FSNamesystem implements Nam
throw new IOException("Block (=" + lastblock + ") not found");
}
}
- INodeFile iFile = ((INode)storedBlock.getBlockCollection()).asFile();
+ //
+ // The implementation of delete operation (see @deleteInternal method)
+ // first removes the file paths from namespace, and delays the removal
+ // of blocks to later time for better performance. When
+ // commitBlockSynchronization (this method) is called in between, the
+ // blockCollection of storedBlock could have been assigned to null by
+ // the delete operation, throw IOException here instead of NPE; if the
+ // file path is already removed from namespace by the delete operation,
+ // throw FileNotFoundException here, so not to proceed to the end of
+ // this method to add a CloseOp to the edit log for an already deleted
+ // file (See HDFS-6825).
+ //
+ BlockCollection blockCollection = storedBlock.getBlockCollection();
+ if (blockCollection == null) {
+ throw new IOException("The blockCollection of " + storedBlock
+ + " is null, likely because the file owning this block was"
+ + " deleted and the block removal is delayed");
+ }
+ INodeFile iFile = ((INode)blockCollection).asFile();
+ if (isFileDeleted(iFile)) {
+ throw new FileNotFoundException("File not found: "
+ + iFile.getFullPathName() + ", likely due to delayed block"
+ + " removal");
+ }
if (!iFile.isUnderConstruction() || storedBlock.isComplete()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Unexpected block (=" + lastblock
@@ -4052,8 +4378,11 @@ public class FSNamesystem implements Nam
// Otherwise fsck will report these blocks as MISSING, especially if the
// blocksReceived from Datanodes take a long time to arrive.
for (int i = 0; i < trimmedTargets.size(); i++) {
- trimmedTargets.get(i).addBlock(
- trimmedStorages.get(i), storedBlock);
+ DatanodeStorageInfo storageInfo =
+ trimmedTargets.get(i).getStorageInfo(trimmedStorages.get(i));
+ if (storageInfo != null) {
+ storageInfo.addBlock(storedBlock);
+ }
}
}
@@ -4069,7 +4398,8 @@ public class FSNamesystem implements Nam
src = closeFileCommitBlocks(iFile, storedBlock);
} else {
// If this commit does not want to close the file, persist blocks
- src = persistBlocks(iFile, false);
+ src = iFile.getFullPathName();
+ persistBlocks(src, iFile, false);
}
} finally {
writeUnlock();
@@ -4087,11 +4417,10 @@ public class FSNamesystem implements Nam
}
/**
- *
- * @param pendingFile
- * @param storedBlock
+ * @param pendingFile open file that needs to be closed
+ * @param storedBlock last block
* @return Path of the file that was closed.
- * @throws IOException
+ * @throws IOException on error
*/
@VisibleForTesting
String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
@@ -4109,21 +4438,6 @@ public class FSNamesystem implements Nam
}
/**
- * Persist the block list for the given file.
- *
- * @param pendingFile
- * @return Path to the given file.
- * @throws IOException
- */
- @VisibleForTesting
- String persistBlocks(INodeFile pendingFile, boolean logRetryCache)
- throws IOException {
- String src = pendingFile.getFullPathName();
- dir.persistBlocks(src, pendingFile, logRetryCache);
- return src;
- }
-
- /**
* Renew the lease(s) held by the given client
*/
void renewLease(String holder) throws IOException {
@@ -4249,7 +4563,7 @@ public class FSNamesystem implements Nam
* @return registration ID
*/
String getRegistrationID() {
- return Storage.getRegistrationID(dir.fsImage.getStorage());
+ return Storage.getRegistrationID(getFSImage().getStorage());
}
/**
@@ -4299,7 +4613,6 @@ public class FSNamesystem implements Nam
/**
* Perform resource checks and cache the results.
- * @throws IOException
*/
void checkAvailableResources() {
Preconditions.checkState(nnResourceChecker != null,
@@ -4308,6 +4621,85 @@ public class FSNamesystem implements Nam
}
/**
+ * Persist the block list for the inode.
+ * @param path
+ * @param file
+ * @param logRetryCache
+ */
+ private void persistBlocks(String path, INodeFile file,
+ boolean logRetryCache) {
+ assert hasWriteLock();
+ Preconditions.checkArgument(file.isUnderConstruction());
+ getEditLog().logUpdateBlocks(path, file, logRetryCache);
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("persistBlocks: " + path
+ + " with " + file.getBlocks().length + " blocks is persisted to" +
+ " the file system");
+ }
+ }
+
+ void incrDeletedFileCount(long count) {
+ NameNode.getNameNodeMetrics().incrFilesDeleted(count);
+ }
+
+ /**
+ * Close file.
+ * @param path
+ * @param file
+ */
+ private void closeFile(String path, INodeFile file) {
+ assert hasWriteLock();
+ waitForLoadingFSImage();
+ // file is closed
+ getEditLog().logCloseFile(path, file);
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("closeFile: "
+ +path+" with "+ file.getBlocks().length
+ +" blocks is persisted to the file system");
+ }
+ }
+
+ /**
+ * Add the given symbolic link to the fs. Record it in the edits log.
+ * @param path
+ * @param target
+ * @param dirPerms
+ * @param createParent
+ * @param logRetryCache
+ * @param dir
+ */
+ private INodeSymlink addSymlink(String path, String target,
+ PermissionStatus dirPerms,
+ boolean createParent, boolean logRetryCache)
+ throws UnresolvedLinkException, FileAlreadyExistsException,
+ QuotaExceededException, SnapshotAccessControlException, AclException {
+ waitForLoadingFSImage();
+
+ final long modTime = now();
+ if (createParent) {
+ final String parent = new Path(path).getParent().toString();
+ if (!mkdirsRecursively(parent, dirPerms, true, modTime)) {
+ return null;
+ }
+ }
+ final String userName = dirPerms.getUserName();
+ long id = allocateNewInodeId();
+ INodeSymlink newNode = dir.addSymlink(id, path, target, modTime, modTime,
+ new PermissionStatus(userName, null, FsPermission.getDefault()));
+ if (newNode == null) {
+ NameNode.stateChangeLog.info("addSymlink: failed to add " + path);
+ return null;
+ }
+ getEditLog().logSymlink(path, target, modTime, modTime, newNode,
+ logRetryCache);
+
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("addSymlink: " + path + " is added");
+ }
+ return newNode;
+ }
+
+ /**
* Periodically calls hasAvailableResources of NameNodeResourceChecker, and if
* there are found to be insufficient resources available, causes the NN to
* enter safe mode. If resources are later found to have returned to
@@ -4387,7 +4779,7 @@ public class FSNamesystem implements Nam
}
public FSImage getFSImage() {
- return dir.fsImage;
+ return fsImage;
}
public FSEditLog getEditLog() {
@@ -4550,6 +4942,28 @@ public class FSNamesystem implements Nam
}
}
+ DatanodeStorageReport[] getDatanodeStorageReport(final DatanodeReportType type
+ ) throws AccessControlException, StandbyException {
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.UNCHECKED);
+ readLock();
+ try {
+ checkOperation(OperationCategory.UNCHECKED);
+ final DatanodeManager dm = getBlockManager().getDatanodeManager();
+ final List<DatanodeDescriptor> datanodes = dm.getDatanodeListForReport(type);
+
+ DatanodeStorageReport[] reports = new DatanodeStorageReport[datanodes.size()];
+ for (int i = 0; i < reports.length; i++) {
+ final DatanodeDescriptor d = datanodes.get(i);
+ reports[i] = new DatanodeStorageReport(new DatanodeInfo(d),
+ d.getStorageReports());
+ }
+ return reports;
+ } finally {
+ readUnlock();
+ }
+ }
+
/**
* Save namespace image.
* This will save current namespace into fsimage file and empty edits file.
@@ -4640,6 +5054,21 @@ public class FSNamesystem implements Nam
}
/**
+ * Persist the new block (the last block of the given file).
+ * @param path
+ * @param file
+ */
+ private void persistNewBlock(String path, INodeFile file) {
+ Preconditions.checkArgument(file.isUnderConstruction());
+ getEditLog().logAddBlock(path, file);
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("persistNewBlock: "
+ + path + " with new block " + file.getLastBlock().toString()
+ + ", current total block count is " + file.getBlocks().length);
+ }
+ }
+
+ /**
* SafeModeInfo contains information related to the safe mode.
* <p>
* An instance of {@link SafeModeInfo} is created when the name node
@@ -4666,8 +5095,12 @@ public class FSNamesystem implements Nam
private final double threshold;
/** Safe mode minimum number of datanodes alive */
private final int datanodeThreshold;
- /** Safe mode extension after the threshold. */
- private int extension;
+ /**
+ * Safe mode extension after the threshold.
+ * Make it volatile so that getSafeModeTip can read the latest value
+ * without taking a lock.
+ */
+ private volatile int extension;
/** Min replication required by safe mode. */
private final int safeReplication;
/** threshold for populating needed replication queues */
@@ -4689,8 +5122,12 @@ public class FSNamesystem implements Nam
private int blockReplQueueThreshold;
/** time of the last status printout */
private long lastStatusReport = 0;
- /** Was safemode entered automatically because available resources were low. */
- private boolean resourcesLow = false;
+ /**
+ * Was safemode entered automatically because available resources were low.
+ * Make it volatile so that getSafeModeTip can read the latest value
+ * without taking a lock.
+ */
+ private volatile boolean resourcesLow = false;
/** Should safemode adjust its block totals as blocks come in */
private boolean shouldIncrementallyTrackBlocks = false;
/** counter for tracking startup progress of reported blocks */
@@ -5350,7 +5787,6 @@ public class FSNamesystem implements Nam
/**
* Leave safe mode.
- * @throws IOException
*/
void leaveSafeMode() {
writeLock();
@@ -5366,14 +5802,21 @@ public class FSNamesystem implements Nam
}
String getSafeModeTip() {
- readLock();
- try {
- if (!isInSafeMode()) {
- return "";
- }
+ // There is no need to take readLock.
+ // Don't use isInSafeMode as this.safeMode might be set to null.
+ // after isInSafeMode returns.
+ boolean inSafeMode;
+ SafeModeInfo safeMode = this.safeMode;
+ if (safeMode == null) {
+ inSafeMode = false;
+ } else {
+ inSafeMode = safeMode.isOn();
+ }
+
+ if (!inSafeMode) {
+ return "";
+ } else {
return safeMode.getTurnOffTip();
- } finally {
- readUnlock();
}
}
@@ -5418,7 +5861,7 @@ public class FSNamesystem implements Nam
}
public void processIncrementalBlockReport(final DatanodeID nodeID,
- final String poolId, final StorageReceivedDeletedBlocks srdb)
+ final StorageReceivedDeletedBlocks srdb)
throws IOException {
writeLock();
try {
@@ -5501,7 +5944,7 @@ public class FSNamesystem implements Nam
FsAction parentAccess, FsAction access, FsAction subAccess)
throws AccessControlException, UnresolvedLinkException {
checkPermission(pc, path, doCheckOwner, ancestorAccess,
- parentAccess, access, subAccess, true);
+ parentAccess, access, subAccess, false, true);
}
/**
@@ -5512,14 +5955,14 @@ public class FSNamesystem implements Nam
private void checkPermission(FSPermissionChecker pc,
String path, boolean doCheckOwner, FsAction ancestorAccess,
FsAction parentAccess, FsAction access, FsAction subAccess,
- boolean resolveLink)
+ boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (!pc.isSuperUser()) {
- dir.waitForReady();
+ waitForLoadingFSImage();
readLock();
try {
- pc.checkPermission(path, dir.rootDir, doCheckOwner, ancestorAccess,
- parentAccess, access, subAccess, resolveLink);
+ pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
+ parentAccess, access, subAccess, ignoreEmptyDir, resolveLink);
} finally {
readUnlock();
}
@@ -5549,12 +5992,9 @@ public class FSNamesystem implements Nam
@Override // FSNamesystemMBean
@Metric
public long getFilesTotal() {
- readLock();
- try {
- return this.dir.totalInodes();
- } finally {
- readUnlock();
- }
+ // There is no need to take fSNamesystem's lock as
+ // FSDirectory has its own lock.
+ return this.dir.totalInodes();
}
@Override // FSNamesystemMBean
@@ -5671,7 +6111,6 @@ public class FSNamesystem implements Nam
blockManager.shutdown();
}
}
-
@Override // FSNamesystemMBean
public int getNumLiveDataNodes() {
@@ -5719,6 +6158,15 @@ public class FSNamesystem implements Nam
}
/**
+ * Storages are marked as "content stale" after NN restart or fails over and
+ * before NN receives the first Heartbeat followed by the first Blockreport.
+ */
+ @Override // FSNamesystemMBean
+ public int getNumStaleStorages() {
+ return getBlockManager().getDatanodeManager().getNumStaleStorages();
+ }
+
+ /**
* Sets the current generation stamp for legacy blocks
*/
void setGenerationStampV1(long stamp) {
@@ -5767,7 +6215,7 @@ public class FSNamesystem implements Nam
/**
* Sets the generation stamp that delineates random and sequentially
* allocated block IDs.
- * @param stamp
+ * @param stamp set generation stamp limit to this value
*/
void setGenerationStampV1Limit(long stamp) {
Preconditions.checkState(generationStampV1Limit ==
@@ -5852,7 +6300,6 @@ public class FSNamesystem implements Nam
* Determine whether the block ID was randomly generated (legacy) or
* sequentially generated. The generation stamp value is used to
* make the distinction.
- * @param block
* @return true if the block ID was randomly generated, false otherwise.
*/
boolean isLegacyBlock(Block block) {
@@ -5871,6 +6318,35 @@ public class FSNamesystem implements Nam
return blockId;
}
+ private boolean isFileDeleted(INodeFile file) {
+ // Not in the inodeMap or in the snapshot but marked deleted.
+ if (dir.getInode(file.getId()) == null) {
+ return true;
+ }
+
+ // look at the path hierarchy to see if one parent is deleted by recursive
+ // deletion
+ INode tmpChild = file;
+ INodeDirectory tmpParent = file.getParent();
+ while (true) {
+ if (tmpParent == null ||
+ tmpParent.searchChildren(tmpChild.getLocalNameBytes()) < 0) {
+ return true;
+ }
+ if (tmpParent.isRoot()) {
+ break;
+ }
+ tmpChild = tmpParent;
+ tmpParent = tmpParent.getParent();
+ }
+
+ if (file.isWithSnapshot() &&
+ file.getFileWithSnapshotFeature().isCurrentFileDeleted()) {
+ return true;
+ }
+ return false;
+ }
+
private INodeFile checkUCBlock(ExtendedBlock block,
String clientName) throws IOException {
assert hasWriteLock();
@@ -5887,7 +6363,7 @@ public class FSNamesystem implements Nam
// check file inode
final INodeFile file = ((INode)storedBlock.getBlockCollection()).asFile();
- if (file == null || !file.isUnderConstruction()) {
+ if (file == null || !file.isUnderConstruction() || isFileDeleted(file)) {
throw new IOException("The file " + storedBlock +
" belonged to does not exist or it is not under construction.");
}
@@ -5986,6 +6462,7 @@ public class FSNamesystem implements Nam
+ ", newNodes=" + Arrays.asList(newNodes)
+ ", clientName=" + clientName
+ ")");
+ waitForLoadingFSImage();
writeLock();
boolean success = false;
try {
@@ -6037,7 +6514,7 @@ public class FSNamesystem implements Nam
blockinfo.setExpectedLocations(storages);
String src = pendingFile.getFullPathName();
- dir.persistBlocks(src, pendingFile, logRetryCache);
+ persistBlocks(src, pendingFile, logRetryCache);
}
// rename was successful. If any part of the renamed subtree had
@@ -6048,6 +6525,42 @@ public class FSNamesystem implements Nam
}
/**
+ * Serializes leases.
+ */
+ void saveFilesUnderConstruction(DataOutputStream out,
+ Map<Long, INodeFile> snapshotUCMap) throws IOException {
+ // This is run by an inferior thread of saveNamespace, which holds a read
+ // lock on our behalf. If we took the read lock here, we could block
+ // for fairness if a writer is waiting on the lock.
+ synchronized (leaseManager) {
+ Map<String, INodeFile> nodes = leaseManager.getINodesUnderConstruction();
+ for (Map.Entry<String, INodeFile> entry : nodes.entrySet()) {
+ // TODO: for HDFS-5428, because of rename operations, some
+ // under-construction files that are
+ // in the current fs directory can also be captured in the
+ // snapshotUCMap. We should remove them from the snapshotUCMap.
+ snapshotUCMap.remove(entry.getValue().getId());
+ }
+
+ out.writeInt(nodes.size() + snapshotUCMap.size()); // write the size
+ for (Map.Entry<String, INodeFile> entry : nodes.entrySet()) {
+ FSImageSerialization.writeINodeUnderConstruction(
+ out, entry.getValue(), entry.getKey());
+ }
+ for (Map.Entry<Long, INodeFile> entry : snapshotUCMap.entrySet()) {
+ // for those snapshot INodeFileUC, we use "/.reserved/.inodes/<inodeid>"
+ // as their paths
+ StringBuilder b = new StringBuilder();
+ b.append(FSDirectory.DOT_RESERVED_PATH_PREFIX)
+ .append(Path.SEPARATOR).append(FSDirectory.DOT_INODES_STRING)
+ .append(Path.SEPARATOR).append(entry.getValue().getId());
+ FSImageSerialization.writeINodeUnderConstruction(
+ out, entry.getValue(), b.toString());
+ }
+ }
+ }
+
+ /**
* @return all the under-construction files in the lease map
*/
Map<String, INodeFile> getFilesUnderConstruction() {
@@ -6089,7 +6602,6 @@ public class FSNamesystem implements Nam
* Release (unregister) backup node.
* <p>
* Find and remove the backup stream corresponding to the node.
- * @param registration
* @throws IOException
*/
void releaseBackupNode(NamenodeRegistration registration)
@@ -6136,6 +6648,23 @@ public class FSNamesystem implements Nam
String[] cookieTab) throws IOException {
checkSuperuserPrivilege();
checkOperation(OperationCategory.READ);
+
+ int count = 0;
+ ArrayList<CorruptFileBlockInfo> corruptFiles =
+ new ArrayList<CorruptFileBlockInfo>();
+ if (cookieTab == null) {
+ cookieTab = new String[] { null };
+ }
+
+ // Do a quick check if there are any corrupt files without taking the lock
+ if (blockManager.getMissingBlocksCount() == 0) {
+ if (cookieTab[0] == null) {
+ cookieTab[0] = String.valueOf(getIntCookie(cookieTab[0]));
+ }
+ LOG.info("there are no corrupt file blocks.");
+ return corruptFiles;
+ }
+
readLock();
try {
checkOperation(OperationCategory.READ);
@@ -6144,14 +6673,9 @@ public class FSNamesystem implements Nam
"replication queues have not been initialized.");
}
// print a limited # of corrupt files per call
- int count = 0;
- ArrayList<CorruptFileBlockInfo> corruptFiles = new ArrayList<CorruptFileBlockInfo>();
final Iterator<Block> blkIterator = blockManager.getCorruptReplicaBlockIterator();
- if (cookieTab == null) {
- cookieTab = new String[] { null };
- }
int skip = getIntCookie(cookieTab[0]);
for (int i = 0; i < skip && blkIterator.hasNext(); i++) {
blkIterator.next();
@@ -6225,8 +6749,8 @@ public class FSNamesystem implements Nam
/**
* @param renewer Renewer information
- * @return Token<DelegationTokenIdentifier>
- * @throws IOException
+ * @return delegation toek
+ * @throws IOException on error
*/
Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException {
@@ -6267,10 +6791,10 @@ public class FSNamesystem implements Nam
/**
*
- * @param token delegation token
- * @return New expiryTime of the token
- * @throws InvalidToken
- * @throws IOException
+ * @param token token to renew
+ * @return new expiryTime of the token
+ * @throws InvalidToken if {@code token} is invalid
+ * @throws IOException on other errors
*/
long renewDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException {
@@ -6301,8 +6825,8 @@ public class FSNamesystem implements Nam
/**
*
- * @param token delegation token that needs to be canceled
- * @throws IOException
+ * @param token token to cancel
+ * @throws IOException on error
*/
void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws IOException {
@@ -6322,6 +6846,15 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
+ /**
+ * @param out save state of the secret manager
+ * @param sdPath String storage directory path
+ */
+ void saveSecretManagerStateCompat(DataOutputStream out, String sdPath)
+ throws IOException {
+ dtSecretManager.saveSecretManagerStateCompat(out, sdPath);
+ }
+
SecretManagerState saveSecretManagerState() {
return dtSecretManager.saveSecretManagerState();
}
@@ -6630,7 +7163,7 @@ public class FSNamesystem implements Nam
@Override // NameNodeMXBean
public String getClusterId() {
- return dir.fsImage.getStorage().getClusterID();
+ return getFSImage().getStorage().getClusterID();
}
@Override // NameNodeMXBean
@@ -6873,7 +7406,18 @@ public class FSNamesystem implements Nam
@Override // FSClusterStats
public int getNumDatanodesInService() {
- return getNumLiveDataNodes() - getNumDecomLiveDataNodes();
+ return datanodeStatistics.getNumDatanodesInService();
+ }
+
+ @Override // for block placement strategy
+ public double getInServiceXceiverAverage() {
+ double avgLoad = 0;
+ final int nodes = getNumDatanodesInService();
+ if (nodes != 0) {
+ final int xceivers = datanodeStatistics.getInServiceXceiverCount();
+ avgLoad = (double)xceivers/nodes;
+ }
+ return avgLoad;
}
public SnapshotManager getSnapshotManager() {
@@ -7071,7 +7615,7 @@ public class FSNamesystem implements Nam
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
- SnapshotDiffInfo diffs = null;
+ SnapshotDiffReport diffs;
checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
@@ -7085,13 +7629,11 @@ public class FSNamesystem implements Nam
} finally {
readUnlock();
}
-
+
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
}
- return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
- path, fromSnapshot, toSnapshot,
- Collections.<DiffReportEntry> emptyList());
+ return diffs;
}
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
@@ -7160,7 +7702,7 @@ public class FSNamesystem implements Nam
* Remove a list of INodeDirectorySnapshottable from the SnapshotManager
* @param toRemove the list of INodeDirectorySnapshottable to be removed
*/
- void removeSnapshottableDirs(List<INodeDirectorySnapshottable> toRemove) {
+ void removeSnapshottableDirs(List<INodeDirectory> toRemove) {
if (snapshotManager != null) {
snapshotManager.removeSnapshottable(toRemove);
}
@@ -7213,7 +7755,7 @@ public class FSNamesystem implements Nam
/**
* Update internal state to indicate that a rolling upgrade is in progress.
- * @param startTime start time of the rolling upgrade
+ * @param startTime rolling upgrade start time
*/
void startRollingUpgradeInternal(long startTime)
throws IOException {
@@ -7306,14 +7848,21 @@ public class FSNamesystem implements Nam
returnInfo = finalizeRollingUpgradeInternal(now());
getEditLog().logFinalizeRollingUpgrade(returnInfo.getFinalizeTime());
- getFSImage().saveNamespace(this);
+ if (haEnabled) {
+ // roll the edit log to make sure the standby NameNode can tail
+ getFSImage().rollEditLog();
+ }
+ getFSImage().updateStorageVersion();
getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK,
NameNodeFile.IMAGE);
} finally {
writeUnlock();
}
- // getEditLog().logSync() is not needed since it does saveNamespace
+ if (!haEnabled) {
+ // Sync not needed for ha since the edit was rolled after logging.
+ getEditLog().logSync();
+ }
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "finalizeRollingUpgrade", null, null, null);
@@ -7348,6 +7897,7 @@ public class FSNamesystem implements Nam
cacheManager.waitForRescanIfNeeded();
}
writeLock();
+ String effectiveDirectiveStr = null;
Long result = null;
try {
checkOperation(OperationCategory.WRITE);
@@ -7359,11 +7909,12 @@ public class FSNamesystem implements Nam
throw new IOException("addDirective: you cannot specify an ID " +
"for this operation.");
}
- CacheDirectiveInfo effectiveDirective =
+ CacheDirectiveInfo effectiveDirective =
cacheManager.addDirective(directive, pc, flags);
getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
cacheEntry != null);
result = effectiveDirective.getId();
+ effectiveDirectiveStr = effectiveDirective.toString();
success = true;
} finally {
writeUnlock();
@@ -7371,7 +7922,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCacheDirective", null, null, null);
+ logAuditEvent(success, "addCacheDirective", effectiveDirectiveStr, null, null);
}
RetryCache.setState(cacheEntry, success, result);
}
@@ -7408,7 +7959,8 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "modifyCacheDirective", null, null, null);
+ String idStr = "{id: " + directive.getId().toString() + "}";
+ logAuditEvent(success, "modifyCacheDirective", idStr, directive.toString(), null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7436,7 +7988,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "removeCacheDirective", null, null,
+ String idStr = "{id: " + id.toString() + "}";
+ logAuditEvent(success, "removeCacheDirective", idStr, null,
null);
}
RetryCache.setState(cacheEntry, success);
@@ -7461,7 +8014,7 @@ public class FSNamesystem implements Nam
} finally {
readUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "listCacheDirectives", null, null,
+ logAuditEvent(success, "listCacheDirectives", filter.toString(), null,
null);
}
}
@@ -7478,6 +8031,7 @@ public class FSNamesystem implements Nam
}
writeLock();
boolean success = false;
+ String poolInfoStr = null;
try {
checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
@@ -7488,12 +8042,13 @@ public class FSNamesystem implements Nam
pc.checkSuperuserPrivilege();
}
CachePoolInfo info = cacheManager.addCachePool(req);
+ poolInfoStr = info.toString();
getEditLog().logAddCachePool(info, cacheEntry != null);
success = true;
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCachePool", req.getPoolName(), null, null);
+ logAuditEvent(success, "addCachePool", poolInfoStr, null, null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7526,7 +8081,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "modifyCachePool", req.getPoolName(), null, null);
+ String poolNameStr = "{poolName: " + req.getPoolName() + "}";
+ logAuditEvent(success, "modifyCachePool", poolNameStr, req.toString(), null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7559,7 +8115,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "removeCachePool", cachePoolName, null, null);
+ String poolNameStr = "{poolName: " + cachePoolName + "}";
+ logAuditEvent(success, "removeCachePool", poolNameStr, null, null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7590,7 +8147,7 @@ public class FSNamesystem implements Nam
}
void modifyAclEntries(String src, List<AclEntry> aclSpec) throws IOException {
- aclConfigFlag.checkForApiCall();
+ nnConf.checkAclsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
@@ -7601,7 +8158,8 @@ public class FSNamesystem implements Nam
checkNameNodeSafeMode("Cannot modify ACL entries on " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
- dir.modifyAclEntries(src, aclSpec);
+ List<AclEntry> newAcl = dir.modifyAclEntries(src, aclSpec);
+ getEditLog().logSetAcl(src, newAcl);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -7611,7 +8169,7 @@ public class FSNamesystem implements Nam
}
void removeAclEntries(String src, List<AclEntry> aclSpec) throws IOException {
- aclConfigFlag.checkForApiCall();
+ nnConf.checkAclsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
@@ -7622,7 +8180,8 @@ public class FSNamesystem implements Nam
checkNameNodeSafeMode("Cannot remove ACL entries on " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
- dir.removeAclEntries(src, aclSpec);
+ List<AclEntry> newAcl = dir.removeAclEntries(src, aclSpec);
+ getEditLog().logSetAcl(src, newAcl);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -7632,7 +8191,7 @@ public class FSNamesystem implements Nam
}
void removeDefaultAcl(String src) throws IOException {
- aclConfigFlag.checkForApiCall();
+ nnConf.checkAclsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
@@ -7643,7 +8202,8 @@ public class FSNamesystem implements Nam
checkNameNodeSafeMode("Cannot remove default ACL entries on " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
- dir.removeDefaultAcl(src);
+ List<AclEntry> newAcl = dir.removeDefaultAcl(src);
+ getEditLog().logSetAcl(src, newAcl);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -7653,7 +8213,7 @@ public class FSNamesystem implements Nam
}
void removeAcl(String src) throws IOException {
- aclConfigFlag.checkForApiCall();
+ nnConf.checkAclsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
@@ -7665,6 +8225,7 @@ public class FSNamesystem implements Nam
[... 326 lines stripped ...]