You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/01/13 16:33:30 UTC
[01/14] hadoop git commit: HDFS-7326. Add documentation for hdfs
debug commands (Vijay Bhat via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-EC 82c57eff9 -> 9953e9214
HDFS-7326. Add documentation for hdfs debug commands (Vijay Bhat via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fd8ea16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fd8ea16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fd8ea16
Branch: refs/heads/HDFS-EC
Commit: 0fd8ea169dba10d564784edaf3e637549b756f24
Parents: 6ee81a7
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jan 12 12:16:36 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:16 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../src/site/apt/HDFSCommands.apt.vm | 43 ++++++++++++++++++++
2 files changed, 46 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd8ea16/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1a5a22b..f83ac65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -490,6 +490,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7323. Move the get/setStoragePolicy commands out from dfsadmin.
(jing9 via yliu)
+ HDFS-7326: Add documentation for hdfs debug commands (Vijay Bhat via Colin
+ P. McCabe)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fd8ea16/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 281221d..4cf31ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -464,3 +464,46 @@ HDFS Commands Guide
| -geteditsize | Prints the number of uncheckpointed transactions on
| the NameNode.
*----------------------+------------------------------------------------------+
+
+* Debug Commands
+
+ Useful commands to help administrators debug HDFS issues, like validating
+ block files and calling recoverLease.
+
+** <<<verify>>>
+
+ Verify HDFS metadata and block files. If a block file is specified, we
+ will verify that the checksums in the metadata file match the block
+ file.
+
+ Usage: <<<hdfs dfs verify [-meta <metadata-file>] [-block <block-file>]>>>
+
+*------------------------+----------------------------------------------------+
+|| COMMAND_OPTION | Description
+*------------------------+----------------------------------------------------+
+| -meta <metadata-file> | Absolute path for the metadata file on the local file
+| | system of the data node.
+*------------------------+----------------------------------------------------+
+| -block <block-file> | Optional parameter to specify the absolute path for
+| | the block file on the local file system of the data
+| | node.
+*------------------------+----------------------------------------------------+
+
+
+
+** <<<recoverLease>>>
+
+ Recover the lease on the specified path. The path must reside on an
+ HDFS filesystem. The default number of retries is 1.
+
+ Usage: <<<hdfs dfs recoverLease [-path <path>] [-retries <num-retries>]>>>
+
+*-------------------------------+--------------------------------------------+
+|| COMMAND_OPTION || Description
+*-------------------------------+---------------------------------------------+
+| [-path <path>] | HDFS path for which to recover the lease.
+*-------------------------------+---------------------------------------------+
+| [-retries <num-retries>] | Number of times the client will retry calling
+| | recoverLease. The default number of retries
+| | is 1.
+*-------------------------------+---------------------------------------------+
[06/14] hadoop git commit: YARN-2643. Don't create a new
DominantResourceCalculator on every FairScheduler.allocate call. (kasha via
rkanter)
Posted by zh...@apache.org.
YARN-2643. Don't create a new DominantResourceCalculator on every FairScheduler.allocate call. (kasha via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e213fc1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e213fc1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e213fc1
Branch: refs/heads/HDFS-EC
Commit: 2e213fc1e18d6a2c0caf1f1796e2b1b96d5171e2
Parents: 210f5c5
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Jan 12 17:51:00 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/scheduler/fair/FairScheduler.java | 7 +++++--
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e213fc1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f849960..0afc4cf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -180,6 +180,9 @@ Release 2.7.0 - UNRELEASED
YARN-2957. Create unit test to automatically compare YarnConfiguration
and yarn-default.xml. (rchiang via rkanter)
+ YARN-2643. Don't create a new DominantResourceCalculator on every
+ FairScheduler.allocate call. (kasha via rkanter)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e213fc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 2f0857e..1ace604 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -127,6 +127,8 @@ public class FairScheduler extends
private static final ResourceCalculator RESOURCE_CALCULATOR =
new DefaultResourceCalculator();
+ private static final ResourceCalculator DOMINANT_RESOURCE_CALCULATOR =
+ new DominantResourceCalculator();
// Value that container assignment methods return when a container is
// reserved
@@ -878,7 +880,8 @@ public class FairScheduler extends
@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
- List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ List<ResourceRequest> ask, List<ContainerId> release,
+ List<String> blacklistAdditions, List<String> blacklistRemovals) {
// Make sure this application exists
FSAppAttempt application = getSchedulerApp(appAttemptId);
@@ -889,7 +892,7 @@ public class FairScheduler extends
}
// Sanity check
- SchedulerUtils.normalizeRequests(ask, new DominantResourceCalculator(),
+ SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR,
clusterResource, minimumAllocation, getMaximumResourceCapability(),
incrAllocation);
[12/14] hadoop git commit: HDFS-7056. Snapshot support for truncate.
Contributed by Konstantin Shvachko and Plamen Jeliazkov.
Posted by zh...@apache.org.
HDFS-7056. Snapshot support for truncate. Contributed by Konstantin Shvachko and Plamen Jeliazkov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9953e921
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9953e921
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9953e921
Branch: refs/heads/HDFS-EC
Commit: 9953e9214577dca33a1c635167e706aecad5080a
Parents: 215c2af
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Tue Jan 13 00:24:23 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:18 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 6 +-
.../InterDatanodeProtocolTranslatorPB.java | 5 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 13 +-
.../BlockInfoUnderConstruction.java | 20 +-
.../server/blockmanagement/BlockManager.java | 5 +-
.../server/blockmanagement/DatanodeManager.java | 32 +-
.../hdfs/server/common/HdfsServerConstants.java | 7 -
.../hadoop/hdfs/server/datanode/DataNode.java | 37 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 81 ++-
.../server/namenode/FSDirStatAndListingOp.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 36 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSEditLogOp.java | 22 +
.../hdfs/server/namenode/FSNamesystem.java | 203 +++++--
.../hadoop/hdfs/server/namenode/INode.java | 15 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 165 ++++--
.../server/namenode/NameNodeLayoutVersion.java | 5 +-
.../snapshot/AbstractINodeDiffList.java | 15 +-
.../snapshot/FSImageFormatPBSnapshot.java | 23 +
.../hdfs/server/namenode/snapshot/FileDiff.java | 42 +-
.../server/namenode/snapshot/FileDiffList.java | 98 ++++
.../snapshot/FileWithSnapshotFeature.java | 24 +-
.../server/protocol/BlockRecoveryCommand.java | 21 +-
.../server/protocol/InterDatanodeProtocol.java | 3 +-
.../src/main/proto/InterDatanodeProtocol.proto | 2 +
.../hadoop-hdfs/src/main/proto/fsimage.proto | 1 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 6 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 1 +
.../hdfs/server/datanode/TestBlockRecovery.java | 53 +-
.../impl/TestInterDatanodeProtocol.java | 9 +-
.../TestCommitBlockSynchronization.java | 3 +
.../hdfs/server/namenode/TestFileTruncate.java | 525 +++++++++++++++++--
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5791 -> 5586 bytes
.../src/test/resources/editsStored.xml | 238 ++++-----
39 files changed, 1347 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6da0667..c1c01d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -20,6 +20,8 @@ Trunk (Unreleased)
HDFS-3107. Introduce truncate. (Plamen Jeliazkov via shv)
+ HDFS-7056. Snapshot support for truncate. (Plamen Jeliazkov and shv)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 749f387..cfd1c67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -537,7 +537,7 @@ public interface ClientProtocol {
* @param src existing file
* @param newLength the target size
*
- * @return true if and client does not need to wait for block recovery,
+ * @return true if client does not need to wait for block recovery,
* false if client needs to wait for block recovery.
*
* @throws AccessControlException If access is denied
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
index 087c697..ba0a8fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
@@ -76,12 +76,12 @@ public class InterDatanodeProtocolServerSideTranslatorPB implements
final String storageID;
try {
storageID = impl.updateReplicaUnderRecovery(
- PBHelper.convert(request.getBlock()),
- request.getRecoveryId(), request.getNewLength());
+ PBHelper.convert(request.getBlock()), request.getRecoveryId(),
+ request.getNewBlockId(), request.getNewLength());
} catch (IOException e) {
throw new ServiceException(e);
}
return UpdateReplicaUnderRecoveryResponseProto.newBuilder()
.setStorageUuid(storageID).build();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
index 5174d86..fee62a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
@@ -102,11 +102,12 @@ public class InterDatanodeProtocolTranslatorPB implements
@Override
public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
- long recoveryId, long newLength) throws IOException {
+ long recoveryId, long newBlockId, long newLength) throws IOException {
UpdateReplicaUnderRecoveryRequestProto req =
UpdateReplicaUnderRecoveryRequestProto.newBuilder()
.setBlock(PBHelper.convert(oldBlock))
- .setNewLength(newLength).setRecoveryId(recoveryId).build();
+ .setNewLength(newLength).setNewBlockId(newBlockId)
+ .setRecoveryId(recoveryId).build();
try {
return rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req
).getStorageUuid();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 3f6a7f3..7187838 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -607,16 +607,19 @@ public class PBHelper {
return null;
}
LocatedBlockProto lb = PBHelper.convert((LocatedBlock)b);
- return RecoveringBlockProto.newBuilder().setBlock(lb)
- .setNewGenStamp(b.getNewGenerationStamp())
- .setTruncateFlag(b.getTruncateFlag()).build();
+ RecoveringBlockProto.Builder builder = RecoveringBlockProto.newBuilder();
+ builder.setBlock(lb).setNewGenStamp(b.getNewGenerationStamp());
+ if(b.getNewBlock() != null)
+ builder.setTruncateBlock(PBHelper.convert(b.getNewBlock()));
+ return builder.build();
}
public static RecoveringBlock convert(RecoveringBlockProto b) {
ExtendedBlock block = convert(b.getBlock().getB());
DatanodeInfo[] locs = convert(b.getBlock().getLocsList());
- return new RecoveringBlock(block, locs, b.getNewGenStamp(),
- b.getTruncateFlag());
+ return (b.hasTruncateBlock()) ?
+ new RecoveringBlock(block, locs, PBHelper.convert(b.getTruncateBlock())) :
+ new RecoveringBlock(block, locs, b.getNewGenStamp());
}
public static DatanodeInfoProto.AdminState convert(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
index 28b179d..8a811ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -55,6 +55,11 @@ public class BlockInfoUnderConstruction extends BlockInfo {
private long blockRecoveryId = 0;
/**
+ * The block source to use in the event of copy-on-write truncate.
+ */
+ private Block truncateBlock;
+
+ /**
* ReplicaUnderConstruction contains information about replicas while
* they are under construction.
* The GS, the length and the state of the replica is as reported by
@@ -229,6 +234,15 @@ public class BlockInfoUnderConstruction extends BlockInfo {
return blockRecoveryId;
}
+ /** Get recover block */
+ public Block getTruncateBlock() {
+ return truncateBlock;
+ }
+
+ public void setTruncateBlock(Block recoveryBlock) {
+ this.truncateBlock = recoveryBlock;
+ }
+
/**
* Process the recorded replicas. When about to commit or finish the
* pipeline recovery sort out bad replicas.
@@ -273,11 +287,7 @@ public class BlockInfoUnderConstruction extends BlockInfo {
* make it primary.
*/
public void initializeBlockRecovery(long recoveryId) {
- initializeBlockRecovery(BlockUCState.UNDER_RECOVERY, recoveryId);
- }
-
- public void initializeBlockRecovery(BlockUCState s, long recoveryId) {
- setBlockUCState(s);
+ setBlockUCState(BlockUCState.UNDER_RECOVERY);
blockRecoveryId = recoveryId;
if (replicas.size() == 0) {
NameNode.blockStateChangeLog.warn("BLOCK*"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7e54c0f..0386107 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -700,13 +700,14 @@ public class BlockManager {
* The client is supposed to allocate a new block with the next call.
*
* @param bc file
+ * @param bytesToRemove num of bytes to remove from block
* @return the last block locations if the block is partial or null otherwise
*/
public LocatedBlock convertLastBlockToUnderConstruction(
- BlockCollection bc) throws IOException {
+ BlockCollection bc, long bytesToRemove) throws IOException {
BlockInfo oldBlock = bc.getLastBlock();
if(oldBlock == null ||
- bc.getPreferredBlockSize() == oldBlock.getNumBytes())
+ bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove)
return null;
assert oldBlock == getStoredBlock(oldBlock) :
"last block of the file is not in blocksMap";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 918b8d9..7ef6521 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -1433,26 +1432,37 @@ public class DatanodeManager {
recoveryLocations.add(storages[i]);
}
}
+ // If we are performing a truncate recovery than set recovery fields
+ // to old block.
+ boolean truncateRecovery = b.getTruncateBlock() != null;
+ boolean copyOnTruncateRecovery = truncateRecovery &&
+ b.getTruncateBlock().getBlockId() != b.getBlockId();
+ ExtendedBlock primaryBlock = (copyOnTruncateRecovery) ?
+ new ExtendedBlock(blockPoolId, b.getTruncateBlock()) :
+ new ExtendedBlock(blockPoolId, b);
// If we only get 1 replica after eliminating stale nodes, then choose all
// replicas for recovery and let the primary data node handle failures.
+ DatanodeInfo[] recoveryInfos;
if (recoveryLocations.size() > 1) {
if (recoveryLocations.size() != storages.length) {
LOG.info("Skipped stale nodes for recovery : " +
(storages.length - recoveryLocations.size()));
}
- boolean isTruncate = b.getBlockUCState().equals(
- HdfsServerConstants.BlockUCState.BEING_TRUNCATED);
- brCommand.add(new RecoveringBlock(
- new ExtendedBlock(blockPoolId, b),
- DatanodeStorageInfo.toDatanodeInfos(recoveryLocations),
- b.getBlockRecoveryId(), isTruncate));
+ recoveryInfos =
+ DatanodeStorageInfo.toDatanodeInfos(recoveryLocations);
} else {
// If too many replicas are stale, then choose all replicas to participate
// in block recovery.
- brCommand.add(new RecoveringBlock(
- new ExtendedBlock(blockPoolId, b),
- DatanodeStorageInfo.toDatanodeInfos(storages),
- b.getBlockRecoveryId()));
+ recoveryInfos = DatanodeStorageInfo.toDatanodeInfos(storages);
+ }
+ if(truncateRecovery) {
+ Block recoveryBlock = (copyOnTruncateRecovery) ? b :
+ b.getTruncateBlock();
+ brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos,
+ recoveryBlock));
+ } else {
+ brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos,
+ b.getBlockRecoveryId()));
}
}
return new DatanodeCommand[] { brCommand };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index f2e7ff4..9bba2c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -300,13 +300,6 @@ public final class HdfsServerConstants {
*/
UNDER_RECOVERY,
/**
- * The block is being truncated.<br>
- * When a file is truncated its last block may need to be truncated
- * and needs to go through a recovery procedure,
- * which synchronizes the existing replicas contents.
- */
- BEING_TRUNCATED,
- /**
* The block is committed.<br>
* The client reported that all bytes are written to data-nodes
* with the given generation stamp and block length, but no
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 7f95f33..84528e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2530,14 +2530,16 @@ public class DataNode extends ReconfigurableBase
*/
@Override // InterDatanodeProtocol
public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock,
- final long recoveryId, final long newLength) throws IOException {
+ final long recoveryId, final long newBlockId, final long newLength)
+ throws IOException {
final String storageID = data.updateReplicaUnderRecovery(oldBlock,
- recoveryId, newLength);
+ recoveryId, newBlockId, newLength);
// Notify the namenode of the updated block info. This is important
// for HA, since otherwise the standby node may lose track of the
// block locations until the next block report.
ExtendedBlock newBlock = new ExtendedBlock(oldBlock);
newBlock.setGenerationStamp(recoveryId);
+ newBlock.setBlockId(newBlockId);
newBlock.setNumBytes(newLength);
notifyNamenodeReceivedBlock(newBlock, "", storageID);
return storageID;
@@ -2559,10 +2561,12 @@ public class DataNode extends ReconfigurableBase
this.rInfo = rInfo;
}
- void updateReplicaUnderRecovery(String bpid, long recoveryId, long newLength
- ) throws IOException {
+ void updateReplicaUnderRecovery(String bpid, long recoveryId,
+ long newBlockId, long newLength)
+ throws IOException {
final ExtendedBlock b = new ExtendedBlock(bpid, rInfo);
- storageID = datanode.updateReplicaUnderRecovery(b, recoveryId, newLength);
+ storageID = datanode.updateReplicaUnderRecovery(b, recoveryId, newBlockId,
+ newLength);
}
@Override
@@ -2644,8 +2648,12 @@ public class DataNode extends ReconfigurableBase
final String bpid = block.getBlockPoolId();
DatanodeProtocolClientSideTranslatorPB nn =
getActiveNamenodeForBP(block.getBlockPoolId());
-
+
long recoveryId = rBlock.getNewGenerationStamp();
+ boolean isTruncateRecovery = rBlock.getNewBlock() != null;
+ long blockId = (isTruncateRecovery) ?
+ rBlock.getNewBlock().getBlockId() : block.getBlockId();
+
if (LOG.isDebugEnabled()) {
LOG.debug("block=" + block + ", (length=" + block.getNumBytes()
+ "), syncList=" + syncList);
@@ -2679,7 +2687,7 @@ public class DataNode extends ReconfigurableBase
// Calculate list of nodes that will participate in the recovery
// and the new block size
List<BlockRecord> participatingList = new ArrayList<BlockRecord>();
- final ExtendedBlock newBlock = new ExtendedBlock(bpid, block.getBlockId(),
+ final ExtendedBlock newBlock = new ExtendedBlock(bpid, blockId,
-1, recoveryId);
switch(bestState) {
case FINALIZED:
@@ -2691,10 +2699,7 @@ public class DataNode extends ReconfigurableBase
r.rInfo.getNumBytes() == finalizedLength)
participatingList.add(r);
}
- if(rBlock.getTruncateFlag())
- newBlock.setNumBytes(rBlock.getBlock().getNumBytes());
- else
- newBlock.setNumBytes(finalizedLength);
+ newBlock.setNumBytes(finalizedLength);
break;
case RBW:
case RWR:
@@ -2706,21 +2711,21 @@ public class DataNode extends ReconfigurableBase
participatingList.add(r);
}
}
- if(rBlock.getTruncateFlag())
- newBlock.setNumBytes(rBlock.getBlock().getNumBytes());
- else
- newBlock.setNumBytes(minLength);
+ newBlock.setNumBytes(minLength);
break;
case RUR:
case TEMPORARY:
assert false : "bad replica state: " + bestState;
}
+ if(isTruncateRecovery)
+ newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
List<DatanodeID> failedList = new ArrayList<DatanodeID>();
final List<BlockRecord> successList = new ArrayList<BlockRecord>();
for(BlockRecord r : participatingList) {
try {
- r.updateReplicaUnderRecovery(bpid, recoveryId, newBlock.getNumBytes());
+ r.updateReplicaUnderRecovery(bpid, recoveryId, blockId,
+ newBlock.getNumBytes());
successList.add(r);
} catch (IOException e) {
InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 462ad31..f2dddfd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -418,7 +418,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* @return the ID of storage that stores the block
*/
public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
- long recoveryId, long newLength) throws IOException;
+ long recoveryId, long newBlockId, long newLength) throws IOException;
/**
* add new block pool ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 3eda38e..e62986f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -670,6 +670,12 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId);
final File dstFile = new File(destDir, srcFile.getName());
final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp);
+ return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum);
+ }
+
+ static File[] copyBlockFiles(File srcMeta, File srcFile, File dstMeta,
+ File dstFile, boolean calculateChecksum)
+ throws IOException {
if (calculateChecksum) {
computeChecksum(srcMeta, dstMeta, srcFile);
} else {
@@ -2157,6 +2163,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
public synchronized String updateReplicaUnderRecovery(
final ExtendedBlock oldBlock,
final long recoveryId,
+ final long newBlockId,
final long newlength) throws IOException {
//get replica
final String bpid = oldBlock.getBlockPoolId();
@@ -2189,13 +2196,26 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
//update replica
final FinalizedReplica finalized = updateReplicaUnderRecovery(oldBlock
- .getBlockPoolId(), (ReplicaUnderRecovery) replica, recoveryId, newlength);
- assert finalized.getBlockId() == oldBlock.getBlockId()
- && finalized.getGenerationStamp() == recoveryId
- && finalized.getNumBytes() == newlength
- : "Replica information mismatched: oldBlock=" + oldBlock
- + ", recoveryId=" + recoveryId + ", newlength=" + newlength
- + ", finalized=" + finalized;
+ .getBlockPoolId(), (ReplicaUnderRecovery) replica, recoveryId,
+ newBlockId, newlength);
+
+ boolean copyTruncate = newBlockId != oldBlock.getBlockId();
+ if(!copyTruncate) {
+ assert finalized.getBlockId() == oldBlock.getBlockId()
+ && finalized.getGenerationStamp() == recoveryId
+ && finalized.getNumBytes() == newlength
+ : "Replica information mismatched: oldBlock=" + oldBlock
+ + ", recoveryId=" + recoveryId + ", newlength=" + newlength
+ + ", newBlockId=" + newBlockId + ", finalized=" + finalized;
+ } else {
+ assert finalized.getBlockId() == oldBlock.getBlockId()
+ && finalized.getGenerationStamp() == oldBlock.getGenerationStamp()
+ && finalized.getNumBytes() == oldBlock.getNumBytes()
+ : "Finalized and old information mismatched: oldBlock=" + oldBlock
+ + ", genStamp=" + oldBlock.getGenerationStamp()
+ + ", len=" + oldBlock.getNumBytes()
+ + ", finalized=" + finalized;
+ }
//check replica files after update
checkReplicaFiles(finalized);
@@ -2208,6 +2228,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
String bpid,
ReplicaUnderRecovery rur,
long recoveryId,
+ long newBlockId,
long newlength) throws IOException {
//check recovery id
if (rur.getRecoveryID() != recoveryId) {
@@ -2215,26 +2236,62 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
+ ", rur=" + rur);
}
+ boolean copyOnTruncate = newBlockId > 0L && rur.getBlockId() != newBlockId;
+ File blockFile;
+ File metaFile;
// bump rur's GS to be recovery id
- bumpReplicaGS(rur, recoveryId);
+ if(!copyOnTruncate) {
+ bumpReplicaGS(rur, recoveryId);
+ blockFile = rur.getBlockFile();
+ metaFile = rur.getMetaFile();
+ } else {
+ File[] copiedReplicaFiles =
+ copyReplicaWithNewBlockIdAndGS(rur, bpid, newBlockId, recoveryId);
+ blockFile = copiedReplicaFiles[1];
+ metaFile = copiedReplicaFiles[0];
+ }
//update length
- final File replicafile = rur.getBlockFile();
if (rur.getNumBytes() < newlength) {
throw new IOException("rur.getNumBytes() < newlength = " + newlength
+ ", rur=" + rur);
}
if (rur.getNumBytes() > newlength) {
rur.unlinkBlock(1);
- truncateBlock(replicafile, rur.getMetaFile(), rur.getNumBytes(), newlength);
- // update RUR with the new length
- rur.setNumBytes(newlength);
+ truncateBlock(blockFile, metaFile, rur.getNumBytes(), newlength);
+ if(!copyOnTruncate) {
+ // update RUR with the new length
+ rur.setNumBytes(newlength);
+ } else {
+ // Copying block to a new block with new blockId.
+ // Not truncating original block.
+ ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(
+ newBlockId, recoveryId, rur.getVolume(), blockFile.getParentFile(),
+ newlength);
+ newReplicaInfo.setNumBytes(newlength);
+ volumeMap.add(bpid, newReplicaInfo);
+ finalizeReplica(bpid, newReplicaInfo);
+ }
}
// finalize the block
return finalizeReplica(bpid, rur);
}
+ private File[] copyReplicaWithNewBlockIdAndGS(
+ ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS)
+ throws IOException {
+ String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId;
+ FsVolumeImpl v = volumes.getNextVolume(
+ replicaInfo.getVolume().getStorageType(), replicaInfo.getNumBytes());
+ final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir();
+ final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId);
+ final File dstBlockFile = new File(destDir, blockFileName);
+ final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS);
+ return copyBlockFiles(replicaInfo.getMetaFile(), replicaInfo.getBlockFile(),
+ dstMetaFile, dstBlockFile, true);
+ }
+
@Override // FsDatasetSpi
public synchronized long getReplicaVisibleLength(final ExtendedBlock block)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index dc0fe1f..cb3da19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -424,7 +424,7 @@ class FSDirStatAndListingOp {
fileNode.computeFileSizeNotIncludingLastUcBlock() : size;
loc = fsd.getFSNamesystem().getBlockManager().createLocatedBlocks(
- fileNode.getBlocks(), fileSize, isUc, 0L, size, false,
+ fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false,
inSnapshot, feInfo);
if (loc == null) {
loc = new LocatedBlocks();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 1948099..9c33e06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1093,18 +1093,31 @@ public class FSDirectory implements Closeable {
* Unlike FSNamesystem.truncate, this will not schedule block recovery.
*/
void unprotectedTruncate(String src, String clientName, String clientMachine,
- long newLength, long mtime)
+ long newLength, long mtime, Block truncateBlock)
throws UnresolvedLinkException, QuotaExceededException,
SnapshotAccessControlException, IOException {
INodesInPath iip = getINodesInPath(src, true);
+ INodeFile file = iip.getLastINode().asFile();
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
boolean onBlockBoundary =
unprotectedTruncate(iip, newLength, collectedBlocks, mtime);
if(! onBlockBoundary) {
- getFSNamesystem().prepareFileForWrite(src,
- iip, clientName, clientMachine, false, false);
+ BlockInfo oldBlock = file.getLastBlock();
+ Block tBlk =
+ getFSNamesystem().prepareFileForTruncate(iip,
+ clientName, clientMachine, file.computeFileSize() - newLength,
+ truncateBlock);
+ assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
+ tBlk.getNumBytes() == truncateBlock.getNumBytes() :
+ "Should be the same block.";
+ if(oldBlock.getBlockId() != tBlk.getBlockId() &&
+ !file.isBlockInLatestSnapshot(oldBlock)) {
+ getBlockManager().removeBlockFromMap(oldBlock);
+ }
}
+ assert onBlockBoundary == (truncateBlock == null) :
+ "truncateBlock is null iff on block boundary: " + truncateBlock;
getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
@@ -1123,7 +1136,8 @@ public class FSDirectory implements Closeable {
/**
* Truncate has the following properties:
* 1.) Any block deletions occur now.
- * 2.) INode length is truncated now – clients can only read up to new length.
+ * 2.) INode length is truncated now – new clients can only read up to
+ * the truncated length.
* 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
* 4.) NN will trigger DN truncation recovery and waits for DNs to report.
* 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
@@ -1136,20 +1150,16 @@ public class FSDirectory implements Closeable {
long mtime) throws IOException {
assert hasWriteLock();
INodeFile file = iip.getLastINode().asFile();
+ int latestSnapshot = iip.getLatestSnapshotId();
+ file.recordModification(latestSnapshot, true);
long oldDiskspace = file.diskspaceConsumed();
long remainingLength =
file.collectBlocksBeyondMax(newLength, collectedBlocks);
+ file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
file.setModificationTime(mtime);
updateCount(iip, 0, file.diskspaceConsumed() - oldDiskspace, true);
- // If on block boundary, then return
- long lastBlockDelta = remainingLength - newLength;
- if(lastBlockDelta == 0)
- return true;
- // Set new last block length
- BlockInfo lastBlock = file.getLastBlock();
- assert lastBlock.getNumBytes() - lastBlockDelta > 0 : "wrong block size";
- lastBlock.setNumBytes(lastBlock.getNumBytes() - lastBlockDelta);
- return false;
+ // return whether on a block boundary
+ return (remainingLength - newLength) == 0;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index d32aad9..144be37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -902,13 +903,14 @@ public class FSEditLog implements LogsPurgeable {
* Add truncate file record to edit log
*/
void logTruncate(String src, String clientName, String clientMachine,
- long size, long timestamp) {
+ long size, long timestamp, Block truncateBlock) {
TruncateOp op = TruncateOp.getInstance(cache.get())
.setPath(src)
.setClientName(clientName)
.setClientMachine(clientMachine)
.setNewLength(size)
- .setTimestamp(timestamp);
+ .setTimestamp(timestamp)
+ .setTruncateBlock(truncateBlock);
logEdit(op);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 2ff3b77..1f4d1a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -857,7 +857,8 @@ public class FSEditLogLoader {
case OP_TRUNCATE: {
TruncateOp truncateOp = (TruncateOp) op;
fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
- truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp);
+ truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp,
+ truncateOp.truncateBlock);
break;
}
case OP_SET_STORAGE_POLICY: {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 396fb08..9424156 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -2611,6 +2611,7 @@ public abstract class FSEditLogOp {
String clientMachine;
long newLength;
long timestamp;
+ Block truncateBlock;
private TruncateOp() {
super(OP_TRUNCATE);
@@ -2654,6 +2655,11 @@ public abstract class FSEditLogOp {
return this;
}
+ TruncateOp setTruncateBlock(Block truncateBlock) {
+ this.truncateBlock = truncateBlock;
+ return this;
+ }
+
@Override
void readFields(DataInputStream in, int logVersion) throws IOException {
src = FSImageSerialization.readString(in);
@@ -2661,6 +2667,10 @@ public abstract class FSEditLogOp {
clientMachine = FSImageSerialization.readString(in);
newLength = FSImageSerialization.readLong(in);
timestamp = FSImageSerialization.readLong(in);
+ Block[] blocks =
+ FSImageSerialization.readCompactBlockArray(in, logVersion);
+ assert blocks.length <= 1 : "Truncate op should have 1 or 0 blocks";
+ truncateBlock = (blocks.length == 0) ? null : blocks[0];
}
@Override
@@ -2670,6 +2680,12 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeString(clientMachine, out);
FSImageSerialization.writeLong(newLength, out);
FSImageSerialization.writeLong(timestamp, out);
+ int size = truncateBlock != null ? 1 : 0;
+ Block[] blocks = new Block[size];
+ if (truncateBlock != null) {
+ blocks[0] = truncateBlock;
+ }
+ FSImageSerialization.writeCompactBlockArray(blocks, out);
}
@Override
@@ -2681,6 +2697,8 @@ public abstract class FSEditLogOp {
Long.toString(newLength));
XMLUtils.addSaxString(contentHandler, "TIMESTAMP",
Long.toString(timestamp));
+ if(truncateBlock != null)
+ FSEditLogOp.blockToXml(contentHandler, truncateBlock);
}
@Override
@@ -2690,6 +2708,8 @@ public abstract class FSEditLogOp {
this.clientMachine = st.getValue("CLIENTMACHINE");
this.newLength = Long.parseLong(st.getValue("NEWLENGTH"));
this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
+ if (st.hasChildren("BLOCK"))
+ this.truncateBlock = FSEditLogOp.blockFromXml(st);
}
@Override
@@ -2705,6 +2725,8 @@ public abstract class FSEditLogOp {
builder.append(newLength);
builder.append(", timestamp=");
builder.append(timestamp);
+ builder.append(", truncateBlock=");
+ builder.append(truncateBlock);
builder.append(", opCode=");
builder.append(opCode);
builder.append(", txid=");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c250838..bcc0f5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1837,8 +1837,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
: dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId(), iip);
final LocatedBlocks blocks = blockManager.createLocatedBlocks(
- inode.getBlocks(), fileSize, isUc, offset, length, needBlockToken,
- iip.isSnapshot(), feInfo);
+ inode.getBlocks(iip.getPathSnapshotId()), fileSize,
+ isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo);
// Set caching information for the located blocks.
for (LocatedBlock lb : blocks.getLocatedBlocks()) {
@@ -1912,7 +1912,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Truncation at block boundary is atomic, otherwise it requires
* block recovery to truncate the last block of the file.
*
- * @return true if and client does not need to wait for block recovery,
+ * @return true if client does not need to wait for block recovery,
* false if client needs to wait for block recovery.
*/
boolean truncate(String src, long newLength,
@@ -1974,44 +1974,119 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
dir.checkPathAccess(pc, iip, FsAction.WRITE);
}
INodeFile file = iip.getLastINode().asFile();
- // Data will be lost after truncate occurs so it cannot support snapshots.
- if(file.isInLatestSnapshot(iip.getLatestSnapshotId()))
- throw new HadoopIllegalArgumentException(
- "Cannot truncate file with snapshot.");
// Opening an existing file for write. May need lease recovery.
recoverLeaseInternal(iip, src, clientName, clientMachine, false);
- // Refresh INode as the file could have been closed
- iip = dir.getINodesInPath4Write(src, true);
file = INodeFile.valueOf(iip.getLastINode(), src);
// Truncate length check.
long oldLength = file.computeFileSize();
- if(oldLength == newLength)
+ if(oldLength == newLength) {
return true;
- if(oldLength < newLength)
+ }
+ if(oldLength < newLength) {
throw new HadoopIllegalArgumentException(
"Cannot truncate to a larger file size. Current size: " + oldLength +
", truncate size: " + newLength + ".");
+ }
// Perform INodeFile truncation.
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
boolean onBlockBoundary = dir.truncate(iip, newLength,
collectedBlocks, mtime);
-
+ Block truncateBlock = null;
if(! onBlockBoundary) {
// Open file for write, but don't log into edits
- prepareFileForWrite(src, iip, clientName, clientMachine, false, false);
- file = INodeFile.valueOf(dir.getINode4Write(src), src);
- initializeBlockRecovery(file);
+ long lastBlockDelta = file.computeFileSize() - newLength;
+ assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
+ truncateBlock = prepareFileForTruncate(iip, clientName, clientMachine,
+ lastBlockDelta, null);
}
- getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime);
+ getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime,
+ truncateBlock);
removeBlocks(collectedBlocks);
return onBlockBoundary;
}
- void initializeBlockRecovery(INodeFile inodeFile) throws IOException {
- BlockInfo lastBlock = inodeFile.getLastBlock();
- long recoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(lastBlock));
- ((BlockInfoUnderConstruction)lastBlock).initializeBlockRecovery(
- BlockUCState.BEING_TRUNCATED, recoveryId);
+ /**
+ * Convert current INode to UnderConstruction.
+ * Recreate lease.
+ * Create new block for the truncated copy.
+ * Schedule truncation of the replicas.
+ *
+ * @return the returned block will be written to editLog and passed back into
+ * this method upon loading.
+ */
+ Block prepareFileForTruncate(INodesInPath iip,
+ String leaseHolder,
+ String clientMachine,
+ long lastBlockDelta,
+ Block newBlock)
+ throws IOException {
+ INodeFile file = iip.getLastINode().asFile();
+ String src = iip.getPath();
+ file.recordModification(iip.getLatestSnapshotId());
+ file.toUnderConstruction(leaseHolder, clientMachine);
+ assert file.isUnderConstruction() : "inode should be under construction.";
+ leaseManager.addLease(
+ file.getFileUnderConstructionFeature().getClientName(), src);
+ boolean shouldRecoverNow = (newBlock == null);
+ BlockInfo oldBlock = file.getLastBlock();
+ boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
+ if(newBlock == null) {
+ newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
+ new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
+ nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
+ }
+
+ BlockInfoUnderConstruction truncatedBlockUC;
+ if(shouldCopyOnTruncate) {
+ // Add new truncateBlock into blocksMap and
+ // use oldBlock as a source for copy-on-truncate recovery
+ truncatedBlockUC = new BlockInfoUnderConstruction(newBlock,
+ file.getBlockReplication());
+ truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.setTruncateBlock(oldBlock);
+ file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
+ getBlockManager().addBlockCollection(truncatedBlockUC, file);
+
+ NameNode.stateChangeLog.info("BLOCK* prepareFileForTruncate: "
+ + "Scheduling copy-on-truncate to new size "
+ + truncatedBlockUC.getNumBytes() + " new block " + newBlock
+ + " old block " + truncatedBlockUC.getTruncateBlock());
+ } else {
+ // Use new generation stamp for in-place truncate recovery
+ blockManager.convertLastBlockToUnderConstruction(file, lastBlockDelta);
+ oldBlock = file.getLastBlock();
+ assert !oldBlock.isComplete() : "oldBlock should be under construction";
+ truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
+ truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
+ truncatedBlockUC.getTruncateBlock().setNumBytes(
+ oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.getTruncateBlock().setGenerationStamp(
+ newBlock.getGenerationStamp());
+
+ NameNode.stateChangeLog.debug("BLOCK* prepareFileForTruncate: "
+ + "Scheduling in-place block truncate to new size "
+ + truncatedBlockUC.getTruncateBlock().getNumBytes()
+ + " block=" + truncatedBlockUC);
+ }
+ if(shouldRecoverNow)
+ truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
+
+ // update the quota: use the preferred block size for UC block
+ final long diff =
+ file.getPreferredBlockSize() - truncatedBlockUC.getNumBytes();
+ dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
+ return newBlock;
+ }
+
+ /**
+ * Defines if a replica needs to be copied on truncate or
+ * can be truncated in place.
+ */
+ boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
+ if(!isUpgradeFinalized()) {
+ return true;
+ }
+ return file.isBlockInLatestSnapshot(blk);
}
/**
@@ -2598,7 +2673,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
leaseManager.addLease(
file.getFileUnderConstructionFeature().getClientName(), src);
- LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(file);
+ LocatedBlock ret =
+ blockManager.convertLastBlockToUnderConstruction(file, 0);
if (ret != null) {
// update the quota: use the preferred block size for UC block
final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
@@ -2661,7 +2737,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return false;
}
- private void recoverLeaseInternal(INodesInPath iip,
+ void recoverLeaseInternal(INodesInPath iip,
String src, String holder, String clientMachine, boolean force)
throws IOException {
assert hasWriteLock();
@@ -2723,8 +2799,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} else {
final BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null
- && (lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY ||
- lastBlock.getBlockUCState() == BlockUCState.BEING_TRUNCATED)) {
+ && lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
throw new RecoveryInProgressException("Recovery in progress, file ["
+ src + "], " + "lease owner [" + lease.getHolder() + "]");
} else {
@@ -3942,8 +4017,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new AlreadyBeingCreatedException(message);
case UNDER_CONSTRUCTION:
case UNDER_RECOVERY:
- case BEING_TRUNCATED:
final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock;
+ // determine if last block was intended to be truncated
+ Block recoveryBlock = uc.getTruncateBlock();
+ boolean truncateRecovery = recoveryBlock != null;
+ boolean copyOnTruncate = truncateRecovery &&
+ recoveryBlock.getBlockId() != uc.getBlockId();
+ assert !copyOnTruncate ||
+ recoveryBlock.getBlockId() < uc.getBlockId() &&
+ recoveryBlock.getGenerationStamp() < uc.getGenerationStamp() &&
+ recoveryBlock.getNumBytes() > uc.getNumBytes() :
+ "wrong recoveryBlock";
+
// setup the last block locations from the blockManager if not known
if (uc.getNumExpectedLocations() == 0) {
uc.setExpectedLocations(blockManager.getStorages(lastBlock));
@@ -3964,9 +4049,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// start recovery of the last block for this file
long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
- if (uc.getBlockUCState() != BlockUCState.BEING_TRUNCATED) {
- uc.initializeBlockRecovery(blockRecoveryId);
+ if(copyOnTruncate) {
+ uc.setGenerationStamp(blockRecoveryId);
+ } else if(truncateRecovery) {
+ recoveryBlock.setGenerationStamp(blockRecoveryId);
}
+ uc.initializeBlockRecovery(blockRecoveryId);
leaseManager.renewLease(lease);
// Cannot close file right now, since the last block requires recovery.
// This may potentially cause infinite loop in lease recovery
@@ -4076,11 +4164,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return true;
}
- void commitBlockSynchronization(ExtendedBlock lastblock,
+ void commitBlockSynchronization(ExtendedBlock oldBlock,
long newgenerationstamp, long newlength,
boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
String[] newtargetstorages) throws IOException {
- LOG.info("commitBlockSynchronization(lastblock=" + lastblock
+ LOG.info("commitBlockSynchronization(oldBlock=" + oldBlock
+ ", newgenerationstamp=" + newgenerationstamp
+ ", newlength=" + newlength
+ ", newtargets=" + Arrays.asList(newtargets)
@@ -4099,17 +4187,17 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkNameNodeSafeMode(
"Cannot commitBlockSynchronization while in safe mode");
final BlockInfo storedBlock = getStoredBlock(
- ExtendedBlock.getLocalBlock(lastblock));
+ ExtendedBlock.getLocalBlock(oldBlock));
if (storedBlock == null) {
if (deleteblock) {
// This may be a retry attempt so ignore the failure
// to locate the block.
if (LOG.isDebugEnabled()) {
- LOG.debug("Block (=" + lastblock + ") not found");
+ LOG.debug("Block (=" + oldBlock + ") not found");
}
return;
} else {
- throw new IOException("Block (=" + lastblock + ") not found");
+ throw new IOException("Block (=" + oldBlock + ") not found");
}
}
//
@@ -4136,34 +4224,40 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ iFile.getFullPathName() + ", likely due to delayed block"
+ " removal");
}
- if (!iFile.isUnderConstruction() || storedBlock.isComplete()) {
+ if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) &&
+ iFile.getLastBlock().isComplete()) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Unexpected block (=" + lastblock
+ LOG.debug("Unexpected block (=" + oldBlock
+ ") since the file (=" + iFile.getLocalName()
+ ") is not under construction");
}
return;
}
- long recoveryId =
- ((BlockInfoUnderConstruction)storedBlock).getBlockRecoveryId();
+ BlockInfoUnderConstruction truncatedBlock =
+ (BlockInfoUnderConstruction) iFile.getLastBlock();
+ long recoveryId = truncatedBlock.getBlockRecoveryId();
+ boolean copyTruncate =
+ truncatedBlock.getBlockId() != storedBlock.getBlockId();
if(recoveryId != newgenerationstamp) {
throw new IOException("The recovery id " + newgenerationstamp
+ " does not match current recovery id "
- + recoveryId + " for block " + lastblock);
+ + recoveryId + " for block " + oldBlock);
}
if (deleteblock) {
- Block blockToDel = ExtendedBlock.getLocalBlock(lastblock);
+ Block blockToDel = ExtendedBlock.getLocalBlock(oldBlock);
boolean remove = iFile.removeLastBlock(blockToDel);
if (remove) {
- blockManager.removeBlockFromMap(storedBlock);
+ blockManager.removeBlock(storedBlock);
}
}
else {
// update last block
- storedBlock.setGenerationStamp(newgenerationstamp);
- storedBlock.setNumBytes(newlength);
+ if(!copyTruncate) {
+ storedBlock.setGenerationStamp(newgenerationstamp);
+ storedBlock.setNumBytes(newlength);
+ }
// find the DatanodeDescriptor objects
// There should be no locations in the blockManager till now because the
@@ -4193,7 +4287,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
DatanodeStorageInfo storageInfo =
trimmedTargets.get(i).getStorageInfo(trimmedStorages.get(i));
if (storageInfo != null) {
- storageInfo.addBlock(storedBlock);
+ if(copyTruncate) {
+ storageInfo.addBlock(truncatedBlock);
+ } else {
+ storageInfo.addBlock(storedBlock);
+ }
}
}
}
@@ -4203,11 +4301,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
blockManager.getDatanodeManager().getDatanodeStorageInfos(
trimmedTargets.toArray(new DatanodeID[trimmedTargets.size()]),
trimmedStorages.toArray(new String[trimmedStorages.size()]));
- iFile.setLastBlock(storedBlock, trimmedStorageInfos);
+ if(copyTruncate) {
+ iFile.setLastBlock(truncatedBlock, trimmedStorageInfos);
+ } else {
+ iFile.setLastBlock(storedBlock, trimmedStorageInfos);
+ }
}
if (closeFile) {
- src = closeFileCommitBlocks(iFile, storedBlock);
+ if(copyTruncate) {
+ src = closeFileCommitBlocks(iFile, truncatedBlock);
+ if(!iFile.isBlockInLatestSnapshot(storedBlock)) {
+ blockManager.removeBlock(storedBlock);
+ }
+ } else {
+ src = closeFileCommitBlocks(iFile, storedBlock);
+ }
} else {
// If this commit does not want to close the file, persist blocks
src = iFile.getFullPathName();
@@ -4218,13 +4327,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
getEditLog().logSync();
if (closeFile) {
- LOG.info("commitBlockSynchronization(newblock=" + lastblock
+ LOG.info("commitBlockSynchronization(oldBlock=" + oldBlock
+ ", file=" + src
+ ", newgenerationstamp=" + newgenerationstamp
+ ", newlength=" + newlength
+ ", newtargets=" + Arrays.asList(newtargets) + ") successful");
} else {
- LOG.info("commitBlockSynchronization(" + lastblock + ") successful");
+ LOG.info("commitBlockSynchronization(" + oldBlock + ") successful");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 41b2391..58ef536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -228,7 +228,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
/** Is this inode in the latest snapshot? */
public final boolean isInLatestSnapshot(final int latestSnapshotId) {
- if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) {
+ if (latestSnapshotId == Snapshot.CURRENT_STATE_ID ||
+ latestSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
return false;
}
// if parent is a reference node, parent must be a renamed node. We can
@@ -817,11 +818,15 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* @param toDelete the to-be-deleted block
*/
public void addDeleteBlock(Block toDelete) {
- if (toDelete != null) {
- toDeleteList.add(toDelete);
- }
+ assert toDelete != null : "toDelete is null";
+ toDeleteList.add(toDelete);
}
-
+
+ public void removeDeleteBlock(Block block) {
+ assert block != null : "block is null";
+ toDeleteList.remove(block);
+ }
+
/**
* Clear {@link BlocksMapUpdateInfo#toDeleteList}
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index d1ff2f7..64887e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -24,7 +24,9 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -304,6 +306,11 @@ public class INodeFile extends INodeWithAdditionalFields
@Override
public void recordModification(final int latestSnapshotId)
throws QuotaExceededException {
+ recordModification(latestSnapshotId, false);
+ }
+
+ public void recordModification(final int latestSnapshotId, boolean withBlocks)
+ throws QuotaExceededException {
if (isInLatestSnapshot(latestSnapshotId)
&& !shouldRecordInSrcSnapshot(latestSnapshotId)) {
// the file is in snapshot, create a snapshot feature if it does not have
@@ -312,10 +319,10 @@ public class INodeFile extends INodeWithAdditionalFields
sf = addSnapshotFeature(null);
}
// record self in the diff list if necessary
- sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null);
+ sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null, withBlocks);
}
}
-
+
public FileDiffList getDiffs() {
FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
if (sf != null) {
@@ -415,6 +422,20 @@ public class INodeFile extends INodeWithAdditionalFields
return this.blocks;
}
+ /** @return blocks of the file corresponding to the snapshot. */
+ public BlockInfo[] getBlocks(int snapshot) {
+ if(snapshot == CURRENT_STATE_ID || getDiffs() == null)
+ return getBlocks();
+ FileDiff diff = getDiffs().getDiffById(snapshot);
+ BlockInfo[] snapshotBlocks = diff == null ? getBlocks() : diff.getBlocks();
+ if(snapshotBlocks != null)
+ return snapshotBlocks;
+ // Blocks are not in the current snapshot
+ // Find next snapshot with blocks present or return current file blocks
+ snapshotBlocks = getDiffs().findLaterSnapshotBlocks(diff.getSnapshotId());
+ return (snapshotBlocks == null) ? getBlocks() : snapshotBlocks;
+ }
+
void updateBlockCollection() {
if (blocks != null) {
for(BlockInfo b : blocks) {
@@ -509,13 +530,13 @@ public class INodeFile extends INodeWithAdditionalFields
}
clear();
removedINodes.add(this);
-
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) {
+ sf.getDiffs().destroyAndCollectSnapshotBlocks(collectedBlocks);
sf.clearDiffs();
}
}
-
+
@Override
public String getName() {
// Get the full path name of this inode.
@@ -554,39 +575,23 @@ public class INodeFile extends INodeWithAdditionalFields
@Override
public final ContentSummaryComputationContext computeContentSummary(
final ContentSummaryComputationContext summary) {
- computeContentSummary4Snapshot(summary.getCounts());
- computeContentSummary4Current(summary.getCounts());
- return summary;
- }
-
- private void computeContentSummary4Snapshot(final Content.Counts counts) {
- // file length and diskspace only counted for the latest state of the file
- // i.e. either the current state or the last snapshot
+ final Content.Counts counts = summary.getCounts();
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
- if (sf != null) {
+ if (sf == null) {
+ counts.add(Content.LENGTH, computeFileSize());
+ counts.add(Content.FILE, 1);
+ } else {
final FileDiffList diffs = sf.getDiffs();
final int n = diffs.asList().size();
counts.add(Content.FILE, n);
if (n > 0 && sf.isCurrentFileDeleted()) {
counts.add(Content.LENGTH, diffs.getLast().getFileSize());
- }
-
- if (sf.isCurrentFileDeleted()) {
- final long lastFileSize = diffs.getLast().getFileSize();
- counts.add(Content.DISKSPACE, lastFileSize * getBlockReplication());
+ } else {
+ counts.add(Content.LENGTH, computeFileSize());
}
}
- }
-
- private void computeContentSummary4Current(final Content.Counts counts) {
- FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
- if (sf != null && sf.isCurrentFileDeleted()) {
- return;
- }
-
- counts.add(Content.LENGTH, computeFileSize());
- counts.add(Content.FILE, 1);
counts.add(Content.DISKSPACE, diskspaceConsumed());
+ return summary;
}
/** The same as computeFileSize(null). */
@@ -651,9 +656,36 @@ public class INodeFile extends INodeWithAdditionalFields
return size;
}
+ /**
+ * Compute size consumed by all blocks of the current file,
+ * including blocks in its snapshots.
+ * Use preferred block size for the last block if it is under construction.
+ */
public final long diskspaceConsumed() {
- // use preferred block size for the last block if it is under construction
- return computeFileSize(true, true) * getBlockReplication();
+ FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
+ if(sf == null) {
+ return computeFileSize(true, true) * getBlockReplication();
+ }
+
+ // Collect all distinct blocks
+ long size = 0;
+ Set<Block> allBlocks = new HashSet<Block>(Arrays.asList(getBlocks()));
+ List<FileDiff> diffs = sf.getDiffs().asList();
+ for(FileDiff diff : diffs) {
+ BlockInfo[] diffBlocks = diff.getBlocks();
+ if (diffBlocks != null) {
+ allBlocks.addAll(Arrays.asList(diffBlocks));
+ }
+ }
+ for(Block block : allBlocks) {
+ size += block.getNumBytes();
+ }
+ // check if the last block is under construction
+ BlockInfo lastBlock = getLastBlock();
+ if(lastBlock != null && lastBlock instanceof BlockInfoUnderConstruction) {
+ size += getPreferredBlockSize() - lastBlock.getNumBytes();
+ }
+ return size * getBlockReplication();
}
public final long diskspaceConsumed(int lastSnapshotId) {
@@ -706,7 +738,7 @@ public class INodeFile extends INodeWithAdditionalFields
final BlockInfo[] oldBlocks = getBlocks();
if (oldBlocks == null)
return 0;
- //find the minimum n such that the size of the first n blocks > max
+ // find the minimum n such that the size of the first n blocks > max
int n = 0;
long size = 0;
for(; n < oldBlocks.length && max > size; n++) {
@@ -716,23 +748,78 @@ public class INodeFile extends INodeWithAdditionalFields
return size;
// starting from block n, the data is beyond max.
- // resize the array.
+ // resize the array.
+ truncateBlocksTo(n);
+
+ // collect the blocks beyond max
+ if (collectedBlocks != null) {
+ for(; n < oldBlocks.length; n++) {
+ collectedBlocks.addDeleteBlock(oldBlocks[n]);
+ }
+ }
+ return size;
+ }
+
+ void truncateBlocksTo(int n) {
final BlockInfo[] newBlocks;
if (n == 0) {
newBlocks = BlockInfo.EMPTY_ARRAY;
} else {
newBlocks = new BlockInfo[n];
- System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
+ System.arraycopy(getBlocks(), 0, newBlocks, 0, n);
}
// set new blocks
setBlocks(newBlocks);
+ }
- // collect the blocks beyond max
- if (collectedBlocks != null) {
- for(; n < oldBlocks.length; n++) {
- collectedBlocks.addDeleteBlock(oldBlocks[n]);
- }
+ public void collectBlocksBeyondSnapshot(BlockInfo[] snapshotBlocks,
+ BlocksMapUpdateInfo collectedBlocks) {
+ BlockInfo[] oldBlocks = getBlocks();
+ if(snapshotBlocks == null || oldBlocks == null)
+ return;
+ // Skip blocks in common between the file and the snapshot
+ int n = 0;
+ while(n < oldBlocks.length && n < snapshotBlocks.length &&
+ oldBlocks[n] == snapshotBlocks[n]) {
+ n++;
}
- return size;
+ truncateBlocksTo(n);
+ // Collect the remaining blocks of the file
+ while(n < oldBlocks.length) {
+ collectedBlocks.addDeleteBlock(oldBlocks[n++]);
+ }
+ }
+
+ /** Exclude blocks collected for deletion that belong to a snapshot. */
+ void excludeSnapshotBlocks(int snapshotId,
+ BlocksMapUpdateInfo collectedBlocks) {
+ if(collectedBlocks == null || collectedBlocks.getToDeleteList().isEmpty())
+ return;
+ FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
+ if(sf == null)
+ return;
+ BlockInfo[] snapshotBlocks =
+ getDiffs().findEarlierSnapshotBlocks(snapshotId);
+ if(snapshotBlocks == null)
+ return;
+ List<Block> toDelete = collectedBlocks.getToDeleteList();
+ for(Block blk : snapshotBlocks) {
+ if(toDelete.contains(blk))
+ collectedBlocks.removeDeleteBlock(blk);
+ }
+ }
+
+ /**
+ * @return true if the block is contained in a snapshot or false otherwise.
+ */
+ boolean isBlockInLatestSnapshot(BlockInfo block) {
+ FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
+ if (sf == null || sf.getDiffs() == null)
+ return false;
+ BlockInfo[] snapshotBlocks =
+ getDiffs().findEarlierSnapshotBlocks(getDiffs().getLastSnapshotId());
+ if(snapshotBlocks == null)
+ return false;
+ return Arrays.asList(snapshotBlocks).contains(block);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 512913b..d742c6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -69,8 +69,9 @@ public class NameNodeLayoutVersion {
CREATE_OVERWRITE(-58, "Use single editlog record for " +
"creating file with overwrite"),
XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"),
- BLOCK_STORAGE_POLICY(-60, "Block Storage policy");
-
+ BLOCK_STORAGE_POLICY(-60, "Block Storage policy"),
+ TRUNCATE(-61, "Truncate");
+
private final FeatureInfo info;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index d918495..b330215 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -163,9 +163,12 @@ abstract class AbstractINodeDiffList<N extends INode,
* id, otherwise <=.
* @return The id of the latest snapshot before the given snapshot.
*/
- private final int getPrior(int anchorId, boolean exclusive) {
+ public final int getPrior(int anchorId, boolean exclusive) {
if (anchorId == Snapshot.CURRENT_STATE_ID) {
- return getLastSnapshotId();
+ int last = getLastSnapshotId();
+ if(exclusive && last == anchorId)
+ return Snapshot.NO_SNAPSHOT_ID;
+ return last;
}
final int i = Collections.binarySearch(diffs, anchorId);
if (exclusive) { // must be the one before
@@ -290,10 +293,11 @@ abstract class AbstractINodeDiffList<N extends INode,
}
/** Save the snapshot copy to the latest snapshot. */
- public void saveSelf2Snapshot(int latestSnapshotId, N currentINode,
+ public D saveSelf2Snapshot(int latestSnapshotId, N currentINode,
A snapshotCopy) throws QuotaExceededException {
+ D diff = null;
if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) {
- D diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode);
+ diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode);
if (diff.snapshotINode == null) {
if (snapshotCopy == null) {
snapshotCopy = createSnapshotCopy(currentINode);
@@ -301,6 +305,7 @@ abstract class AbstractINodeDiffList<N extends INode,
diff.saveSnapshotCopy(snapshotCopy);
}
}
+ return diff;
}
@Override
@@ -312,4 +317,4 @@ abstract class AbstractINodeDiffList<N extends INode,
public String toString() {
return getClass().getSimpleName() + ": " + diffs;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 86ba03d..7ac85cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -36,6 +36,10 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -229,6 +233,20 @@ public class FSImageFormatPBSnapshot {
FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
pbf.getFileSize());
+ List<BlockProto> bpl = pbf.getBlocksList();
+ BlockInfo[] blocks = new BlockInfo[bpl.size()];
+ for(int j = 0, e = bpl.size(); j < e; ++j) {
+ Block blk = PBHelper.convert(bpl.get(j));
+ BlockInfo storedBlock = fsn.getBlockManager().getStoredBlock(blk);
+ if(storedBlock == null) {
+ storedBlock = fsn.getBlockManager().addBlockCollection(
+ new BlockInfo(blk, copy.getFileReplication()), file);
+ }
+ blocks[j] = storedBlock;
+ }
+ if(blocks.length > 0) {
+ diff.setBlocks(blocks);
+ }
diffs.addFirst(diff);
}
file.addSnapshotFeature(diffs);
@@ -472,6 +490,11 @@ public class FSImageFormatPBSnapshot {
SnapshotDiffSection.FileDiff.Builder fb = SnapshotDiffSection.FileDiff
.newBuilder().setSnapshotId(diff.getSnapshotId())
.setFileSize(diff.getFileSize());
+ if(diff.getBlocks() != null) {
+ for(Block block : diff.getBlocks()) {
+ fb.addBlocks(PBHelper.convert(block));
+ }
+ }
INodeFileAttributes copy = diff.snapshotINode;
if (copy != null) {
fb.setName(ByteString.copyFrom(copy.getLocalNameBytes()))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
index 919ab56..7b52dc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -37,10 +39,13 @@ public class FileDiff extends
/** The file size at snapshot creation time. */
private final long fileSize;
+ /** A copy of the INodeFile block list. Used in truncate. */
+ private BlockInfo[] blocks;
FileDiff(int snapshotId, INodeFile file) {
super(snapshotId, null, null);
fileSize = file.computeFileSize();
+ blocks = null;
}
/** Constructor used by FSImage loading */
@@ -48,20 +53,40 @@ public class FileDiff extends
FileDiff posteriorDiff, long fileSize) {
super(snapshotId, snapshotINode, posteriorDiff);
this.fileSize = fileSize;
+ blocks = null;
}
/** @return the file size in the snapshot. */
public long getFileSize() {
return fileSize;
}
-
+
+ /**
+ * Copy block references into the snapshot
+ * up to the current {@link #fileSize}.
+ * Should be done only once.
+ */
+ public void setBlocks(BlockInfo[] blocks) {
+ if(this.blocks != null)
+ return;
+ int numBlocks = 0;
+ for(long s = 0; numBlocks < blocks.length && s < fileSize; numBlocks++)
+ s += blocks[numBlocks].getNumBytes();
+ this.blocks = Arrays.copyOf(blocks, numBlocks);
+ }
+
+ public BlockInfo[] getBlocks() {
+ return blocks;
+ }
+
@Override
Quota.Counts combinePosteriorAndCollectBlocks(INodeFile currentINode,
FileDiff posterior, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- return currentINode.getFileWithSnapshotFeature()
- .updateQuotaAndCollectBlocks(currentINode, posterior, collectedBlocks,
- removedINodes);
+ FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature();
+ assert sf != null : "FileWithSnapshotFeature is null";
+ return sf.updateQuotaAndCollectBlocks(
+ currentINode, posterior, collectedBlocks, removedINodes);
}
@Override
@@ -91,4 +116,13 @@ public class FileDiff extends
.updateQuotaAndCollectBlocks(currentINode, this, collectedBlocks,
removedINodes);
}
+
+ public void destroyAndCollectSnapshotBlocks(
+ BlocksMapUpdateInfo collectedBlocks) {
+ if(blocks == null || collectedBlocks == null)
+ return;
+ for(BlockInfo blk : blocks)
+ collectedBlocks.addDeleteBlock(blk);
+ blocks = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index b0a973d..07652f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -17,6 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
@@ -33,4 +40,95 @@ public class FileDiffList extends
INodeFileAttributes createSnapshotCopy(INodeFile currentINode) {
return new INodeFileAttributes.SnapshotCopy(currentINode);
}
+
+ public void destroyAndCollectSnapshotBlocks(
+ BlocksMapUpdateInfo collectedBlocks) {
+ for(FileDiff d : asList())
+ d.destroyAndCollectSnapshotBlocks(collectedBlocks);
+ }
+
+ public void saveSelf2Snapshot(int latestSnapshotId, INodeFile iNodeFile,
+ INodeFileAttributes snapshotCopy, boolean withBlocks)
+ throws QuotaExceededException {
+ final FileDiff diff =
+ super.saveSelf2Snapshot(latestSnapshotId, iNodeFile, snapshotCopy);
+ if(withBlocks) // Store blocks if this is the first update
+ diff.setBlocks(iNodeFile.getBlocks());
+ }
+
+ public BlockInfo[] findEarlierSnapshotBlocks(int snapshotId) {
+ assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
+ if(snapshotId == Snapshot.CURRENT_STATE_ID) {
+ return null;
+ }
+ List<FileDiff> diffs = this.asList();
+ int i = Collections.binarySearch(diffs, snapshotId);
+ BlockInfo[] blocks = null;
+ for(i = i >= 0 ? i : -i; i < diffs.size(); i--) {
+ blocks = diffs.get(i).getBlocks();
+ if(blocks != null) {
+ break;
+ }
+ }
+ return blocks;
+ }
+
+ public BlockInfo[] findLaterSnapshotBlocks(int snapshotId) {
+ assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
+ if(snapshotId == Snapshot.CURRENT_STATE_ID) {
+ return null;
+ }
+ List<FileDiff> diffs = this.asList();
+ int i = Collections.binarySearch(diffs, snapshotId);
+ BlockInfo[] blocks = null;
+ for(i = i >= 0 ? i+1 : -i-1; i < diffs.size(); i++) {
+ blocks = diffs.get(i).getBlocks();
+ if(blocks != null) {
+ break;
+ }
+ }
+ return blocks;
+ }
+
+ /**
+ * Copy blocks from the removed snapshot into the previous snapshot
+ * up to the file length of the latter.
+ * Collect unused blocks of the removed snapshot.
+ */
+ void combineAndCollectSnapshotBlocks(INodeFile file,
+ FileDiff removed,
+ BlocksMapUpdateInfo collectedBlocks,
+ List<INode> removedINodes) {
+ BlockInfo[] removedBlocks = removed.getBlocks();
+ if(removedBlocks == null) {
+ FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
+ assert sf != null : "FileWithSnapshotFeature is null";
+ if(sf.isCurrentFileDeleted())
+ sf.collectBlocksAndClear(file, collectedBlocks, removedINodes);
+ return;
+ }
+ int p = getPrior(removed.getSnapshotId(), true);
+ FileDiff earlierDiff = p == Snapshot.NO_SNAPSHOT_ID ? null : getDiffById(p);
+ // Copy blocks to the previous snapshot if not set already
+ if(earlierDiff != null)
+ earlierDiff.setBlocks(removedBlocks);
+ BlockInfo[] earlierBlocks =
+ (earlierDiff == null ? new BlockInfo[]{} : earlierDiff.getBlocks());
+ // Find later snapshot (or file itself) with blocks
+ BlockInfo[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
+ laterBlocks = (laterBlocks==null) ? file.getBlocks() : laterBlocks;
+ // Skip blocks, which belong to either the earlier or the later lists
+ int i = 0;
+ for(; i < removedBlocks.length; i++) {
+ if(i < earlierBlocks.length && removedBlocks[i] == earlierBlocks[i])
+ continue;
+ if(i < laterBlocks.length && removedBlocks[i] == laterBlocks[i])
+ continue;
+ break;
+ }
+ // Collect the remaining blocks of the file
+ while(i < removedBlocks.length) {
+ collectedBlocks.addDeleteBlock(removedBlocks[i++]);
+ }
+ }
}
[10/14] hadoop git commit: HADOOP-11473. test-patch says "-1 overall"
even when all checks are +1 (Jason Lowe via raviprak)
Posted by zh...@apache.org.
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
(Jason Lowe via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/565a4947
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/565a4947
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/565a4947
Branch: refs/heads/HDFS-EC
Commit: 565a494743434b1853c13572d2f535e0fd7f3c57
Parents: 3124609
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Mon Jan 12 14:20:11 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
dev-support/test-patch.sh | 1 -
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565a4947/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index ece3ddf..b0fbb80 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -992,7 +992,6 @@ if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
runTests
(( RESULT = RESULT + $? ))
fi
-(( RESULT = RESULT + $? ))
JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
$JIRA_COMMENT_FOOTER"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565a4947/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 679989a..a9300c2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -148,6 +148,9 @@ Trunk (Unreleased)
BUG FIXES
+ HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
+ (Jason Lowe via raviprak)
+
HADOOP-9451. Fault single-layer config if node group topology is enabled.
(Junping Du via llu)
[03/14] hadoop git commit: YARN-2957. Create unit test to
automatically compare YarnConfiguration and yarn-default.xml. (rchiang via
rkanter)
Posted by zh...@apache.org.
YARN-2957. Create unit test to automatically compare YarnConfiguration and yarn-default.xml. (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ee81a7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ee81a7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ee81a7a
Branch: refs/heads/HDFS-EC
Commit: 6ee81a7aee240a2b5ae45401b7a118f346f15948
Parents: 3e29d9b
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Jan 12 11:57:57 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:16 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/conf/TestYarnConfigurationFields.java | 106 +++++++++++++++++++
2 files changed, 109 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ee81a7a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 153760c..f849960 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -177,6 +177,9 @@ Release 2.7.0 - UNRELEASED
YARN-2956. Added missing links in YARN documentation. (Masatake Iwasaki via
jianhe)
+ YARN-2957. Create unit test to automatically compare YarnConfiguration
+ and yarn-default.xml. (rchiang via rkanter)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ee81a7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
new file mode 100644
index 0000000..9075d9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.conf;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Unit test class to compare
+ * {@link org.apache.hadoop.yarn.conf.YarnConfiguration} and
+ * yarn-default.xml for missing properties. Currently only throws an error
+ * if the class is missing a property.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
+
+ @Override
+ public void initializeMemberVariables() {
+ xmlFilename = new String("yarn-default.xml");
+ configurationClasses = new Class[] { YarnConfiguration.class };
+
+
+ // Allocate for usage
+ configurationPropsToSkipCompare = new HashSet<String>();
+
+ // Set error modes
+ errorIfMissingConfigProps = true;
+ errorIfMissingXmlProps = false;
+
+ // Specific properties to skip
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_IPC_CLIENT_FACTORY_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_IPC_SERVER_FACTORY_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_IPC_RPC_IMPL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_RM_SCHEDULER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration
+ .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
+
+ // Allocate for usage
+ xmlPropsToSkipCompare = new HashSet<String>();
+ xmlPrefixToSkipCompare = new HashSet<String>();
+
+ // Should probably be moved from yarn-default.xml to mapred-default.xml
+ xmlPropsToSkipCompare.add("mapreduce.job.hdfs-servers");
+ xmlPropsToSkipCompare.add("mapreduce.job.jar");
+
+ // Possibly obsolete, but unable to verify 100%
+ xmlPropsToSkipCompare.add("yarn.nodemanager.aux-services.mapreduce_shuffle.class");
+ xmlPropsToSkipCompare.add("yarn.resourcemanager.container.liveness-monitor.interval-ms");
+
+ // Used in the XML file as a variable reference internal to the XML file
+ xmlPropsToSkipCompare.add("yarn.nodemanager.hostname");
+ xmlPropsToSkipCompare.add("yarn.timeline-service.hostname");
+
+ // Currently defined in TimelineAuthenticationFilterInitializer
+ xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication");
+
+ // Currently defined in RegistryConstants
+ xmlPrefixToSkipCompare.add("hadoop.registry");
+ }
+
+}
[07/14] hadoop git commit: HDFS-7533. Datanode sometimes does not
shutdown on receiving upgrade shutdown command. Contributed by Eric Payne.
Posted by zh...@apache.org.
HDFS-7533. Datanode sometimes does not shutdown on receiving upgrade shutdown command. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31246094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31246094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31246094
Branch: refs/heads/HDFS-EC
Commit: 3124609442a749dc24e185737b5148e9558cb352
Parents: 0fd8ea1
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Jan 12 15:36:55 2015 -0600
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +++++++---
.../hdfs/server/datanode/TestDataNodeExit.java | 16 ++++++++++++++++
3 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31246094/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f83ac65..05cb745 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -673,6 +673,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7596. NameNode should prune dead storages from storageMap.
(Arpit Agarwal via cnauroth)
+ HDFS-7533. Datanode sometimes does not shutdown on receiving upgrade
+ shutdown command (Eric Payne via kihwal)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31246094/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 899a729..d5d8429 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1618,9 +1618,13 @@ public class DataNode extends ReconfigurableBase
// in order to avoid any further acceptance of requests, but the peers
// for block writes are not closed until the clients are notified.
if (dataXceiverServer != null) {
- xserver.sendOOBToPeers();
- ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();
- this.dataXceiverServer.interrupt();
+ try {
+ xserver.sendOOBToPeers();
+ ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();
+ this.dataXceiverServer.interrupt();
+ } catch (Throwable e) {
+ // Ignore, since the out of band messaging is advisory.
+ }
}
// Interrupt the checkDiskErrorThread and terminate it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31246094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
index 9d59496..c067b07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
@@ -32,6 +33,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
/**
* Tests if DataNode process exits if all Block Pool services exit.
@@ -88,4 +90,18 @@ public class TestDataNodeExit {
stopBPServiceThreads(2, dn);
assertFalse("DataNode should exit", dn.isDatanodeUp());
}
+
+ @Test
+ public void testSendOOBToPeers() throws Exception {
+ DataNode dn = cluster.getDataNodes().get(0);
+ DataXceiverServer spyXserver = Mockito.spy(dn.getXferServer());
+ NullPointerException e = new NullPointerException();
+ Mockito.doThrow(e).when(spyXserver).sendOOBToPeers();
+ dn.xserver = spyXserver;
+ try {
+ dn.shutdown();
+ } catch (Throwable t) {
+ fail("DataNode shutdown should not have thrown exception " + t);
+ }
+ }
}
[08/14] hadoop git commit: HADOOP-11465. Fix findbugs warnings in
hadoop-gridmix. (Contributed by Varun Saxena)
Posted by zh...@apache.org.
HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Contributed by Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b61ee333
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b61ee333
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b61ee333
Branch: refs/heads/HDFS-EC
Commit: b61ee333a52455b9dc02768ab2e235d33fdbfe8a
Parents: 2e213fc
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jan 12 21:22:58 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../dev-support/findbugs-exclude.xml | 7 ++++
.../gridmix/CompressionEmulationUtil.java | 8 +++--
.../gridmix/DistributedCacheEmulator.java | 10 ++++--
.../mapred/gridmix/GenerateDistCacheData.java | 6 +++-
.../apache/hadoop/mapred/gridmix/Gridmix.java | 36 ++++++++++----------
.../hadoop/mapred/gridmix/PseudoLocalFs.java | 2 +-
.../hadoop/mapred/gridmix/Statistics.java | 26 +++++++++++---
8 files changed, 68 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9300c2..59e6d80 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job
(Akshay Radia via brandonli)
+ HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via
+ Arpit Agarwal)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
index 92458d4..cb043c4 100644
--- a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
@@ -30,4 +30,11 @@
<Bug pattern="REC_CATCH_EXCEPTION"/>
<Bug code="REC"/>
</Match>
+
+ <!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/>
+ <Field name="returnValue"/>
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
index 526878c..bd7e878 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -97,7 +98,9 @@ class CompressionEmulationUtil {
private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE =
new CompressionRatioLookupTable();
-
+
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
/**
* This is a {@link Mapper} implementation for generating random text data.
* It uses {@link RandomTextDataGenerator} for generating text data and the
@@ -133,7 +136,8 @@ class CompressionEmulationUtil {
String randomKey = rtg.getRandomWord();
String randomValue = rtg.getRandomWord();
context.write(new Text(randomKey), new Text(randomValue));
- bytes -= (randomValue.getBytes().length + randomKey.getBytes().length);
+ bytes -= (randomValue.getBytes(charsetUTF8).length +
+ randomKey.getBytes(charsetUTF8).length);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
index ad3b113..d30a51d 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -112,6 +113,8 @@ class DistributedCacheEmulator {
Configuration conf; // gridmix configuration
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
// Pseudo local file system where local FS based distributed cache files are
// created by gridmix.
FileSystem pseudoLocalFs = null;
@@ -436,9 +439,10 @@ class DistributedCacheEmulator {
for (Iterator it = dcFiles.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry)it.next();
LongWritable fileSize =
- new LongWritable(Long.valueOf(entry.getValue().toString()));
+ new LongWritable(Long.parseLong(entry.getValue().toString()));
BytesWritable filePath =
- new BytesWritable(entry.getKey().toString().getBytes());
+ new BytesWritable(
+ entry.getKey().toString().getBytes(charsetUTF8));
byteCount += fileSize.get();
bytesSync += fileSize.get();
@@ -515,7 +519,7 @@ class DistributedCacheEmulator {
// local FS based distributed cache file.
// Create this file on the pseudo local FS.
String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString();
- long fileSize = Long.valueOf(fileSizes[i]);
+ long fileSize = Long.parseLong(fileSizes[i]);
Path mappedLocalFilePath =
PseudoLocalFs.generateFilePath(fileId, fileSize)
.makeQualified(pseudoLocalFs.getUri(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
index 0512746..4a75cde 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.gridmix;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob {
*/
static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
public GenerateDistCacheData(Configuration conf) throws IOException {
super(conf, 0L, JOB_NAME);
}
@@ -152,7 +155,8 @@ class GenerateDistCacheData extends GridmixJob {
public void map(LongWritable key, BytesWritable value, Context context)
throws IOException, InterruptedException {
- String fileName = new String(value.getBytes(), 0, value.getLength());
+ String fileName = new String(value.getBytes(), 0,
+ value.getLength(), charsetUTF8);
Path path = new Path(fileName);
FSDataOutputStream dos =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 8ac590b..4386bc1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -729,40 +729,40 @@ public class Gridmix extends Configured implements Tool {
out.println(" -users <usersResourceURI> : URI that contains the users list.");
out.println("Configuration parameters:");
out.println(" General parameters:");
- out.printf(" %-48s : Output directory\n", GRIDMIX_OUT_DIR);
- out.printf(" %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
- out.printf(" %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
- out.printf(" %-48s : User resolution class\n", GRIDMIX_USR_RSV);
- out.printf(" %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
+ out.printf(" %-48s : Output directory%n", GRIDMIX_OUT_DIR);
+ out.printf(" %-48s : Submitting threads%n", GRIDMIX_SUB_THR);
+ out.printf(" %-48s : Queued job desc%n", GRIDMIX_QUE_DEP);
+ out.printf(" %-48s : User resolution class%n", GRIDMIX_USR_RSV);
+ out.printf(" %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
out.println(" Parameters related to job submission:");
- out.printf(" %-48s : Default queue\n",
+ out.printf(" %-48s : Default queue%n",
GridmixJob.GRIDMIX_DEFAULT_QUEUE);
- out.printf(" %-48s : Enable/disable using queues in trace\n",
+ out.printf(" %-48s : Enable/disable using queues in trace%n",
GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
- out.printf(" %-48s : Job submission policy (%s)\n",
+ out.printf(" %-48s : Job submission policy (%s)%n",
GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
out.println(" Parameters specific for LOADJOB:");
- out.printf(" %-48s : Key fraction of rec\n",
+ out.printf(" %-48s : Key fraction of rec%n",
AvgRecordFactory.GRIDMIX_KEY_FRC);
out.println(" Parameters specific for SLEEPJOB:");
- out.printf(" %-48s : Whether to ignore reduce tasks\n",
+ out.printf(" %-48s : Whether to ignore reduce tasks%n",
SleepJob.SLEEPJOB_MAPTASK_ONLY);
- out.printf(" %-48s : Number of fake locations for map tasks\n",
+ out.printf(" %-48s : Number of fake locations for map tasks%n",
JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
- out.printf(" %-48s : Maximum map task runtime in mili-sec\n",
+ out.printf(" %-48s : Maximum map task runtime in mili-sec%n",
SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
- out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
+ out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n",
SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
out.println(" Parameters specific for STRESS submission throttling policy:");
- out.printf(" %-48s : jobs vs task-tracker ratio\n",
+ out.printf(" %-48s : jobs vs task-tracker ratio%n",
StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
- out.printf(" %-48s : maps vs map-slot ratio\n",
+ out.printf(" %-48s : maps vs map-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
- out.printf(" %-48s : reduces vs reduce-slot ratio\n",
+ out.printf(" %-48s : reduces vs reduce-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
- out.printf(" %-48s : map-slot share per job\n",
+ out.printf(" %-48s : map-slot share per job%n",
StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
- out.printf(" %-48s : reduce-slot share per job\n",
+ out.printf(" %-48s : reduce-slot share per job%n",
StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
index 497108a..d7ef563 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
@@ -124,7 +124,7 @@ class PseudoLocalFs extends FileSystem {
} else {
String[] parts = path.toUri().getPath().split("\\.");
try {
- fileSize = Long.valueOf(parts[parts.length - 1]);
+ fileSize = Long.parseLong(parts[parts.length - 1]);
valid = (fileSize >= 0);
} catch (NumberFormatException e) {
valid = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
index 322d755..915788b 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
@@ -119,7 +119,23 @@ public class Statistics implements Component<Statistics.JobStats> {
}
return new JobStats(maps, reds, job);
}
-
+
+ private static void addToNumMapsSubmitted(int numMaps) {
+ numMapsSubmitted += numMaps;
+ }
+
+ private static void addToNumReducesSubmitted(int numReduces) {
+ numReducesSubmitted += numReduces;
+ }
+
+ private static void subtractFromNumMapsSubmitted(int numMaps) {
+ numMapsSubmitted -= numMaps;
+ }
+
+ private static void subtractFromNumReducesSubmitted(int numReduces) {
+ numReducesSubmitted -= numReduces;
+ }
+
/**
* Add a submitted job for monitoring.
*/
@@ -131,8 +147,8 @@ public class Statistics implements Component<Statistics.JobStats> {
return;
}
submittedJobsMap.put(seq, stats);
- numMapsSubmitted += stats.getNoOfMaps();
- numReducesSubmitted += stats.getNoOfReds();
+ addToNumMapsSubmitted(stats.getNoOfMaps());
+ addToNumReducesSubmitted(stats.getNoOfReds());
}
/**
@@ -156,8 +172,8 @@ public class Statistics implements Component<Statistics.JobStats> {
}
// update the total number of submitted map/reduce task count
- numMapsSubmitted -= stat.getNoOfMaps();
- numReducesSubmitted -= stat.getNoOfReds();
+ subtractFromNumMapsSubmitted(stat.getNoOfMaps());
+ subtractFromNumReducesSubmitted(stat.getNoOfReds());
completedJobsInCurrentInterval++;
//check if we have reached the maximum level of job completions.
[11/14] hadoop git commit: HDFS-7056. Snapshot support for truncate.
Contributed by Konstantin Shvachko and Plamen Jeliazkov.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 16f534f..e348231 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -154,18 +155,19 @@ public class FileWithSnapshotFeature implements INode.Feature {
AclStorage.removeAclFeature(aclFeature);
}
}
-
- collectBlocksAndClear(file, collectedBlocks, removedINodes);
-
+
+ getDiffs().combineAndCollectSnapshotBlocks(
+ file, removed, collectedBlocks, removedINodes);
+
long dsDelta = oldDiskspace - file.diskspaceConsumed();
return Quota.Counts.newInstance(0, dsDelta);
}
-
+
/**
* If some blocks at the end of the block list no longer belongs to
* any inode, collect them and update the block list.
*/
- private void collectBlocksAndClear(final INodeFile file,
+ public void collectBlocksAndClear(final INodeFile file,
final BlocksMapUpdateInfo info, final List<INode> removedINodes) {
// check if everything is deleted.
if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) {
@@ -174,13 +176,19 @@ public class FileWithSnapshotFeature implements INode.Feature {
}
// find max file size.
final long max;
+ FileDiff diff = getDiffs().getLast();
if (isCurrentFileDeleted()) {
- final FileDiff last = getDiffs().getLast();
- max = last == null? 0: last.getFileSize();
+ max = diff == null? 0: diff.getFileSize();
} else {
max = file.computeFileSize();
}
- file.collectBlocksBeyondMax(max, info);
+ // Collect blocks that should be deleted
+ FileDiff last = diffs.getLast();
+ BlockInfo[] snapshotBlocks = last == null ? null : last.getBlocks();
+ if(snapshotBlocks == null)
+ file.collectBlocksBeyondMax(max, info);
+ else
+ file.collectBlocksBeyondSnapshot(snapshotBlocks, info);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
index c512038..ced3296 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -53,8 +54,8 @@ public class BlockRecoveryCommand extends DatanodeCommand {
@InterfaceAudience.Private
@InterfaceStability.Evolving
public static class RecoveringBlock extends LocatedBlock {
- private boolean truncate;
private final long newGenerationStamp;
+ private final Block recoveryBlock;
/**
* Create RecoveringBlock.
@@ -62,15 +63,17 @@ public class BlockRecoveryCommand extends DatanodeCommand {
public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) {
super(b, locs, -1, false); // startOffset is unknown
this.newGenerationStamp = newGS;
+ this.recoveryBlock = null;
}
/**
- * RecoveryingBlock with truncate option.
+ * Create RecoveringBlock with copy-on-truncate option.
*/
- public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS,
- boolean truncate) {
- this(b, locs, newGS);
- this.truncate = truncate;
+ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs,
+ Block recoveryBlock) {
+ super(b, locs, -1, false); // startOffset is unknown
+ this.newGenerationStamp = recoveryBlock.getGenerationStamp();
+ this.recoveryBlock = recoveryBlock;
}
/**
@@ -82,10 +85,10 @@ public class BlockRecoveryCommand extends DatanodeCommand {
}
/**
- * Return whether to truncate the block to the ExtendedBlock's length.
+ * Return the new block.
*/
- public boolean getTruncateFlag() {
- return truncate;
+ public Block getNewBlock() {
+ return recoveryBlock;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
index 62915b4..72cb0c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
@@ -67,5 +67,6 @@ public interface InterDatanodeProtocol {
* Update replica with the new generation stamp and length.
*/
String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId,
- long newLength) throws IOException;
+ long newBlockId, long newLength)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
index 47f79be..1a21777 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
@@ -59,6 +59,8 @@ message UpdateReplicaUnderRecoveryRequestProto {
required ExtendedBlockProto block = 1; // Block identifier
required uint64 recoveryId = 2; // New genstamp of the replica
required uint64 newLength = 3; // New length of the replica
+ // New blockId for copy (truncate), default is 0.
+ optional uint64 newBlockId = 4 [default = 0];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 588f6c8..643a034 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -270,6 +270,7 @@ message SnapshotDiffSection {
optional uint64 fileSize = 2;
optional bytes name = 3;
optional INodeSection.INodeFile snapshotCopy = 4;
+ repeated BlockProto blocks = 5;
}
message DiffEntry {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index d989c0a..97906b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -554,9 +554,9 @@ enum ReplicaStateProto {
* Block that needs to be recovered with at a given location
*/
message RecoveringBlockProto {
- required uint64 newGenStamp = 1; // New genstamp post recovery
- required LocatedBlockProto block = 2; // Block to be recovered
- optional bool truncateFlag = 3; // Block needs to be truncated
+ required uint64 newGenStamp = 1; // New genstamp post recovery
+ required LocatedBlockProto block = 2; // Block to be recovered
+ optional BlockProto truncateBlock = 3; // New block for recovery (truncate)
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index e4834d6..7a4960e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1239,7 +1239,7 @@ public class TestReplicationPolicy {
when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
.thenReturn(ucBlock);
- bm.convertLastBlockToUnderConstruction(mbc);
+ bm.convertLastBlockToUnderConstruction(mbc, 0L);
// Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
// from QUEUE_VERY_UNDER_REPLICATED.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index e03b756..78eedf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -1106,6 +1106,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override // FsDatasetSpi
public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
long recoveryId,
+ long newBlockId,
long newlength) {
// Caller does not care about the exact Storage UUID returned.
return datanodeUuid;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index 987b480..9bf5e52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -56,7 +56,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -219,10 +218,10 @@ public class TestBlockRecovery {
syncList.add(record1);
syncList.add(record2);
- when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
- anyLong())).thenReturn("storage1");
- when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
- anyLong())).thenReturn("storage2");
+ when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
+ anyLong(), anyLong())).thenReturn("storage1");
+ when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
+ anyLong(), anyLong())).thenReturn("storage2");
dn.syncBlock(rBlock, syncList);
}
@@ -245,8 +244,10 @@ public class TestBlockRecovery {
InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
- verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
+ verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
// two finalized replicas have different length
replica1 = new ReplicaRecoveryInfo(BLOCK_ID,
@@ -284,8 +285,10 @@ public class TestBlockRecovery {
InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
- verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
+ verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
// rbw replica has a different length from the finalized one
replica1 = new ReplicaRecoveryInfo(BLOCK_ID,
@@ -297,9 +300,10 @@ public class TestBlockRecovery {
dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
verify(dn2, never()).updateReplicaUnderRecovery(
- block, RECOVERY_ID, REPLICA_LEN1);
+ block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
/**
@@ -323,9 +327,10 @@ public class TestBlockRecovery {
InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
verify(dn2, never()).updateReplicaUnderRecovery(
- block, RECOVERY_ID, REPLICA_LEN1);
+ block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
// rbw replica has a different length from the finalized one
replica1 = new ReplicaRecoveryInfo(BLOCK_ID,
@@ -337,9 +342,10 @@ public class TestBlockRecovery {
dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+ REPLICA_LEN1);
verify(dn2, never()).updateReplicaUnderRecovery(
- block, RECOVERY_ID, REPLICA_LEN1);
+ block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
/**
@@ -362,8 +368,8 @@ public class TestBlockRecovery {
long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
- verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
+ verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
/**
@@ -385,9 +391,9 @@ public class TestBlockRecovery {
InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
verify(dn2, never()).updateReplicaUnderRecovery(
- block, RECOVERY_ID, REPLICA_LEN1);
+ block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
/**
@@ -411,8 +417,8 @@ public class TestBlockRecovery {
long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
- verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
- verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
+ verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
+ verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
private Collection<RecoveringBlock> initRecoveringBlocks() throws IOException {
@@ -513,7 +519,7 @@ public class TestBlockRecovery {
}
DataNode spyDN = spy(dn);
doThrow(new IOException()).when(spyDN).updateReplicaUnderRecovery(
- block, RECOVERY_ID, block.getNumBytes());
+ block, RECOVERY_ID, BLOCK_ID, block.getNumBytes());
try {
spyDN.syncBlock(rBlock, initBlockRecords(spyDN));
fail("Sync should fail");
@@ -634,7 +640,8 @@ public class TestBlockRecovery {
recoveryInitResult.get());
dataNode.updateReplicaUnderRecovery(block.getBlock(), block.getBlock()
- .getGenerationStamp() + 1, block.getBlockSize());
+ .getGenerationStamp() + 1, block.getBlock().getBlockId(),
+ block.getBlockSize());
} finally {
if (null != cluster) {
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
index 65a5176..3609684 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
@@ -198,7 +198,8 @@ public class TestInterDatanodeProtocol {
//verify updateBlock
ExtendedBlock newblock = new ExtendedBlock(b.getBlockPoolId(),
b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
- idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
+ idp.updateReplicaUnderRecovery(b, recoveryId, b.getBlockId(),
+ newblock.getNumBytes());
checkMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
@@ -368,7 +369,8 @@ public class TestInterDatanodeProtocol {
.getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp());
try {
//update should fail
- fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, newlength);
+ fsdataset.updateReplicaUnderRecovery(tmp, recoveryid,
+ tmp.getBlockId(), newlength);
Assert.fail();
} catch(IOException ioe) {
System.out.println("GOOD: getting " + ioe);
@@ -377,7 +379,8 @@ public class TestInterDatanodeProtocol {
//update
final String storageID = fsdataset.updateReplicaUnderRecovery(
- new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, newlength);
+ new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid,
+ rri.getBlockId(), newlength);
assertTrue(storageID != null);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index d0502b3..eae65cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -71,6 +71,7 @@ public class TestCommitBlockSynchronization {
doReturn(true).when(file).isUnderConstruction();
doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
+ doReturn(blockInfo).when(file).getLastBlock();
doReturn("").when(namesystemSpy).closeFileCommitBlocks(
any(INodeFile.class), any(BlockInfo.class));
doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog();
@@ -105,6 +106,7 @@ public class TestCommitBlockSynchronization {
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
.getStoredBlock(any(Block.class));
+ doReturn(completedBlockInfo).when(file).getLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.commitBlockSynchronization(
@@ -176,6 +178,7 @@ public class TestCommitBlockSynchronization {
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
.getStoredBlock(any(Block.class));
+ doReturn(completedBlockInfo).when(file).getLastBlock();
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index ba9d04e..1f854d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -18,14 +18,22 @@
package org.apache.hadoop.hdfs.server.namenode;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.net.InetAddress;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -39,14 +47,14 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -57,6 +65,7 @@ public class TestFileTruncate {
GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
GenericTestUtils.setLogLevel(FSEditLogLoader.LOG, Level.ALL);
}
+ static final Log LOG = LogFactory.getLog(TestFileTruncate.class);
static final int BLOCK_SIZE = 4;
static final short REPLICATION = 3;
static final int DATANODE_NUM = 3;
@@ -129,6 +138,287 @@ public class TestFileTruncate {
fs.delete(parent, true);
}
+ @Test
+ public void testSnapshotWithAppendTruncate() throws IOException {
+ testSnapshotWithAppendTruncate(0, 1, 2);
+ testSnapshotWithAppendTruncate(0, 2, 1);
+ testSnapshotWithAppendTruncate(1, 0, 2);
+ testSnapshotWithAppendTruncate(1, 2, 0);
+ testSnapshotWithAppendTruncate(2, 0, 1);
+ testSnapshotWithAppendTruncate(2, 1, 0);
+ }
+
+ /**
+ * Create three snapshots with appended and truncated file.
+ * Delete snapshots in the specified order and verify that
+ * remaining snapshots are still readable.
+ */
+ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
+ FSDirectory fsDir = cluster.getNamesystem().getFSDirectory();
+ Path parent = new Path("/test");
+ fs.mkdirs(parent);
+ fs.setQuota(parent, 100, 1000);
+ fs.allowSnapshot(parent);
+ String truncateFile = "testSnapshotWithAppendTruncate";
+ final Path src = new Path(parent, truncateFile);
+ int[] length = new int[4];
+ length[0] = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+ DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L);
+ Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock();
+ Path[] snapshotFiles = new Path[4];
+
+ // Diskspace consumed should be 10 bytes * 3. [blk 1,2,3]
+ ContentSummary contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(30L));
+
+ // Add file to snapshot and append
+ String[] ss = new String[] {"ss0", "ss1", "ss2", "ss3"};
+ Path snapshotDir = fs.createSnapshot(parent, ss[0]);
+ snapshotFiles[0] = new Path(snapshotDir, truncateFile);
+ length[1] = length[2] = length[0] + BLOCK_SIZE + 1;
+ DFSTestUtil.appendFile(fs, src, BLOCK_SIZE + 1);
+ Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock()
+ .getBlock().getLocalBlock();
+
+ // Diskspace consumed should be 15 bytes * 3. [blk 1,2,3,4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(45L));
+
+ // Create another snapshot without changes
+ snapshotDir = fs.createSnapshot(parent, ss[1]);
+ snapshotFiles[1] = new Path(snapshotDir, truncateFile);
+
+ // Create another snapshot and append
+ snapshotDir = fs.createSnapshot(parent, ss[2]);
+ snapshotFiles[2] = new Path(snapshotDir, truncateFile);
+ DFSTestUtil.appendFile(fs, src, BLOCK_SIZE -1 + BLOCK_SIZE / 2);
+ Block appendedBlk = getLocatedBlocks(src).getLastLocatedBlock()
+ .getBlock().getLocalBlock();
+
+ // Diskspace consumed should be 20 bytes * 3. [blk 1,2,3,4,5]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(60L));
+
+ // Truncate to block boundary
+ int newLength = length[0] + BLOCK_SIZE / 2;
+ boolean isReady = fs.truncate(src, newLength);
+ assertTrue("Recovery is not expected.", isReady);
+ assertFileLength(snapshotFiles[2], length[2]);
+ assertFileLength(snapshotFiles[1], length[1]);
+ assertFileLength(snapshotFiles[0], length[0]);
+ assertBlockNotPresent(appendedBlk);
+
+ // Diskspace consumed should be 16 bytes * 3. [blk 1,2,3 SS:4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+ // Truncate full block again
+ newLength = length[0] - BLOCK_SIZE / 2;
+ isReady = fs.truncate(src, newLength);
+ assertTrue("Recovery is not expected.", isReady);
+ assertFileLength(snapshotFiles[2], length[2]);
+ assertFileLength(snapshotFiles[1], length[1]);
+ assertFileLength(snapshotFiles[0], length[0]);
+
+ // Diskspace consumed should be 16 bytes * 3. [blk 1,2 SS:3,4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+ // Truncate half of the last block
+ newLength -= BLOCK_SIZE / 2;
+ isReady = fs.truncate(src, newLength);
+ assertFalse("Recovery is expected.", isReady);
+ checkBlockRecovery(src);
+ assertFileLength(snapshotFiles[2], length[2]);
+ assertFileLength(snapshotFiles[1], length[1]);
+ assertFileLength(snapshotFiles[0], length[0]);
+ Block replacedBlk = getLocatedBlocks(src).getLastLocatedBlock()
+ .getBlock().getLocalBlock();
+
+ // Diskspace consumed should be 16 bytes * 3. [blk 1,6 SS:2,3,4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(54L));
+
+ snapshotDir = fs.createSnapshot(parent, ss[3]);
+ snapshotFiles[3] = new Path(snapshotDir, truncateFile);
+ length[3] = newLength;
+
+ // Delete file. Should still be able to read snapshots
+ int numINodes = fsDir.getInodeMapSize();
+ isReady = fs.delete(src, false);
+ assertTrue("Delete failed.", isReady);
+ assertFileLength(snapshotFiles[3], length[3]);
+ assertFileLength(snapshotFiles[2], length[2]);
+ assertFileLength(snapshotFiles[1], length[1]);
+ assertFileLength(snapshotFiles[0], length[0]);
+ assertEquals("Number of INodes should not change",
+ numINodes, fsDir.getInodeMapSize());
+
+ fs.deleteSnapshot(parent, ss[3]);
+
+ assertBlockExists(firstBlk);
+ assertBlockExists(lastBlk);
+ assertBlockNotPresent(replacedBlk);
+
+ // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+ // delete snapshots in the specified order
+ fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
+ assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
+ assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+ assertBlockExists(firstBlk);
+ assertBlockExists(lastBlk);
+ assertEquals("Number of INodes should not change",
+ numINodes, fsDir.getInodeMapSize());
+
+ // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+ fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+ assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+ assertBlockExists(firstBlk);
+ contentSummary = fs.getContentSummary(parent);
+ if(fs.exists(snapshotFiles[0])) {
+ // Diskspace consumed should be 0 bytes * 3. [SS:1,2,3]
+ assertBlockNotPresent(lastBlk);
+ assertThat(contentSummary.getSpaceConsumed(), is(36L));
+ } else {
+ // Diskspace consumed should be 48 bytes * 3. [SS:1,2,3,4]
+ assertThat(contentSummary.getSpaceConsumed(), is(48L));
+ }
+ assertEquals("Number of INodes should not change",
+ numINodes, fsDir .getInodeMapSize());
+
+ fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+ assertBlockNotPresent(firstBlk);
+ assertBlockNotPresent(lastBlk);
+
+ // Diskspace consumed should be 0 bytes * 3. []
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(0L));
+ assertNotEquals("Number of INodes should change",
+ numINodes, fsDir.getInodeMapSize());
+ }
+
+ /**
+ * Create three snapshots with file truncated 3 times.
+ * Delete snapshots in the specified order and verify that
+ * remaining snapshots are still readable.
+ */
+ @Test
+ public void testSnapshotWithTruncates() throws IOException {
+ testSnapshotWithTruncates(0, 1, 2);
+ testSnapshotWithTruncates(0, 2, 1);
+ testSnapshotWithTruncates(1, 0, 2);
+ testSnapshotWithTruncates(1, 2, 0);
+ testSnapshotWithTruncates(2, 0, 1);
+ testSnapshotWithTruncates(2, 1, 0);
+ }
+
+ void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
+ Path parent = new Path("/test");
+ fs.mkdirs(parent);
+ fs.setQuota(parent, 100, 1000);
+ fs.allowSnapshot(parent);
+ String truncateFile = "testSnapshotWithTruncates";
+ final Path src = new Path(parent, truncateFile);
+ int[] length = new int[3];
+ length[0] = 3 * BLOCK_SIZE;
+ DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L);
+ Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock();
+ Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock()
+ .getBlock().getLocalBlock();
+ Path[] snapshotFiles = new Path[3];
+
+ // Diskspace consumed should be 12 bytes * 3. [blk 1,2,3]
+ ContentSummary contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(36L));
+
+ // Add file to snapshot and append
+ String[] ss = new String[] {"ss0", "ss1", "ss2"};
+ Path snapshotDir = fs.createSnapshot(parent, ss[0]);
+ snapshotFiles[0] = new Path(snapshotDir, truncateFile);
+ length[1] = 2 * BLOCK_SIZE;
+ boolean isReady = fs.truncate(src, 2 * BLOCK_SIZE);
+ assertTrue("Recovery is not expected.", isReady);
+
+ // Diskspace consumed should be 12 bytes * 3. [blk 1,2 SS:3]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(36L));
+ snapshotDir = fs.createSnapshot(parent, ss[1]);
+ snapshotFiles[1] = new Path(snapshotDir, truncateFile);
+
+ // Create another snapshot with truncate
+ length[2] = BLOCK_SIZE + BLOCK_SIZE / 2;
+ isReady = fs.truncate(src, BLOCK_SIZE + BLOCK_SIZE / 2);
+ assertFalse("Recovery is expected.", isReady);
+ checkBlockRecovery(src);
+ snapshotDir = fs.createSnapshot(parent, ss[2]);
+ snapshotFiles[2] = new Path(snapshotDir, truncateFile);
+ assertFileLength(snapshotFiles[0], length[0]);
+ assertBlockExists(lastBlk);
+
+ // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(42L));
+
+ fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
+ assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
+ assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+ assertFileLength(src, length[2]);
+ assertBlockExists(firstBlk);
+
+ contentSummary = fs.getContentSummary(parent);
+ if(fs.exists(snapshotFiles[0])) {
+ // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+ assertThat(contentSummary.getSpaceConsumed(), is(42L));
+ assertBlockExists(lastBlk);
+ } else {
+ // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2]
+ assertThat(contentSummary.getSpaceConsumed(), is(30L));
+ assertBlockNotPresent(lastBlk);
+ }
+
+ fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+ assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+ assertFileLength(src, length[2]);
+ assertBlockExists(firstBlk);
+
+ contentSummary = fs.getContentSummary(parent);
+ if(fs.exists(snapshotFiles[0])) {
+ // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+ assertThat(contentSummary.getSpaceConsumed(), is(42L));
+ assertBlockExists(lastBlk);
+ } else if(fs.exists(snapshotFiles[1])) {
+ // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2]
+ assertThat(contentSummary.getSpaceConsumed(), is(30L));
+ assertBlockNotPresent(lastBlk);
+ } else {
+ // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:]
+ assertThat(contentSummary.getSpaceConsumed(), is(18L));
+ assertBlockNotPresent(lastBlk);
+ }
+
+ fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+ assertFileLength(src, length[2]);
+ assertBlockExists(firstBlk);
+
+ // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:]
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(18L));
+ assertThat(contentSummary.getLength(), is(6L));
+
+ fs.delete(src, false);
+ assertBlockNotPresent(firstBlk);
+
+ // Diskspace consumed should be 0 bytes * 3. []
+ contentSummary = fs.getContentSummary(parent);
+ assertThat(contentSummary.getSpaceConsumed(), is(0L));
+ }
+
/**
* Failure / recovery test for truncate.
* In this failure the DNs fail to recover the blocks and the NN triggers
@@ -159,8 +449,6 @@ public class TestFileTruncate {
boolean isReady = fs.truncate(p, newLength);
assertThat("truncate should have triggered block recovery.",
isReady, is(false));
- FileStatus fileStatus = fs.getFileStatus(p);
- assertThat(fileStatus.getLen(), is((long) newLength));
boolean recoveryTriggered = false;
for(int i = 0; i < RECOVERY_ATTEMPTS; i++) {
@@ -168,8 +456,6 @@ public class TestFileTruncate {
NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(),
p.toUri().getPath());
if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
- cluster.startDataNodes(conf, DATANODE_NUM, true,
- HdfsServerConstants.StartupOption.REGULAR, null);
recoveryTriggered = true;
break;
}
@@ -177,6 +463,9 @@ public class TestFileTruncate {
}
assertThat("lease recovery should have occurred in ~" +
SLEEP * RECOVERY_ATTEMPTS + " ms.", recoveryTriggered, is(true));
+ cluster.startDataNodes(conf, DATANODE_NUM, true,
+ StartupOption.REGULAR, null);
+ cluster.waitActive();
checkBlockRecovery(p);
@@ -184,10 +473,10 @@ public class TestFileTruncate {
.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
HdfsConstants.LEASE_HARDLIMIT_PERIOD);
- fileStatus = fs.getFileStatus(p);
+ FileStatus fileStatus = fs.getFileStatus(p);
assertThat(fileStatus.getLen(), is((long) newLength));
- AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ checkFullFile(p, newLength, contents);
fs.delete(p, false);
}
@@ -198,10 +487,9 @@ public class TestFileTruncate {
public void testTruncateEditLogLoad() throws IOException {
int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
int toTruncate = 1;
-
+ final String s = "/testTruncateEditLogLoad";
+ final Path p = new Path(s);
byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
-
- final Path p = new Path("/testTruncateEditLogLoad");
writeContents(contents, startingFileSize, p);
int newLength = startingFileSize - toTruncate;
@@ -209,54 +497,183 @@ public class TestFileTruncate {
assertThat("truncate should have triggered block recovery.",
isReady, is(false));
- checkBlockRecovery(p);
-
cluster.restartNameNode();
+ String holder = UserGroupInformation.getCurrentUser().getUserName();
+ cluster.getNamesystem().recoverLease(s, holder, "");
+
+ checkBlockRecovery(p);
+
FileStatus fileStatus = fs.getFileStatus(p);
assertThat(fileStatus.getLen(), is((long) newLength));
- AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ checkFullFile(p, newLength, contents);
fs.delete(p, false);
}
/**
+ * Upgrade, RollBack, and restart test for Truncate.
+ */
+ @Test
+ public void testUpgradeAndRestart() throws IOException {
+ Path parent = new Path("/test");
+ fs.mkdirs(parent);
+ fs.setQuota(parent, 100, 1000);
+ fs.allowSnapshot(parent);
+ String truncateFile = "testUpgrade";
+ final Path p = new Path(parent, truncateFile);
+ int startingFileSize = 2 * BLOCK_SIZE;
+ int toTruncate = 1;
+ byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+ writeContents(contents, startingFileSize, p);
+
+ Path snapshotDir = fs.createSnapshot(parent, "ss0");
+ Path snapshotFile = new Path(snapshotDir, truncateFile);
+
+ int newLengthBeforeUpgrade = startingFileSize - toTruncate;
+ boolean isReady = fs.truncate(p, newLengthBeforeUpgrade);
+ assertThat("truncate should have triggered block recovery.",
+ isReady, is(false));
+
+ checkBlockRecovery(p);
+
+ checkFullFile(p, newLengthBeforeUpgrade, contents);
+ assertFileLength(snapshotFile, startingFileSize);
+ long totalBlockBefore = cluster.getNamesystem().getBlocksTotal();
+
+ restartCluster(StartupOption.UPGRADE);
+
+ assertThat("SafeMode should be OFF",
+ cluster.getNamesystem().isInSafeMode(), is(false));
+ assertThat("NameNode should be performing upgrade.",
+ cluster.getNamesystem().isUpgradeFinalized(), is(false));
+ FileStatus fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade));
+
+ int newLengthAfterUpgrade = newLengthBeforeUpgrade - toTruncate;
+ Block oldBlk = getLocatedBlocks(p).getLastLocatedBlock()
+ .getBlock().getLocalBlock();
+ isReady = fs.truncate(p, newLengthAfterUpgrade);
+ assertThat("truncate should have triggered block recovery.",
+ isReady, is(false));
+ fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLengthAfterUpgrade));
+ assertThat("Should copy on truncate during upgrade",
+ getLocatedBlocks(p).getLastLocatedBlock().getBlock()
+ .getLocalBlock().getBlockId(), is(not(equalTo(oldBlk.getBlockId()))));
+
+ checkBlockRecovery(p);
+
+ checkFullFile(p, newLengthAfterUpgrade, contents);
+ assertThat("Total block count should be unchanged from copy-on-truncate",
+ cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+
+ restartCluster(StartupOption.ROLLBACK);
+
+ assertThat("File does not exist " + p, fs.exists(p), is(true));
+ fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade));
+ checkFullFile(p, newLengthBeforeUpgrade, contents);
+ assertThat("Total block count should be unchanged from rolling back",
+ cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+
+ restartCluster(StartupOption.REGULAR);
+ assertThat("Total block count should be unchanged from start-up",
+ cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+ checkFullFile(p, newLengthBeforeUpgrade, contents);
+ assertFileLength(snapshotFile, startingFileSize);
+
+ // empty edits and restart
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ cluster.restartNameNode(true);
+ assertThat("Total block count should be unchanged from start-up",
+ cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+ checkFullFile(p, newLengthBeforeUpgrade, contents);
+ assertFileLength(snapshotFile, startingFileSize);
+
+ fs.deleteSnapshot(parent, "ss0");
+ fs.delete(parent, true);
+ assertThat("File " + p + " shouldn't exist", fs.exists(p), is(false));
+ }
+
+ /**
* Check truncate recovery.
*/
@Test
- public void testTruncateLastBlock() throws IOException {
+ public void testTruncateRecovery() throws IOException {
FSNamesystem fsn = cluster.getNamesystem();
-
- String src = "/file";
+ String client = "client";
+ String clientMachine = "clientMachine";
+ Path parent = new Path("/test");
+ String src = "/test/testTruncateRecovery";
Path srcPath = new Path(src);
byte[] contents = AppendTestUtil.initBuffer(BLOCK_SIZE);
writeContents(contents, BLOCK_SIZE, srcPath);
- INodeFile inode = fsn.getFSDirectory().getINode(src).asFile();
- long oldGenstamp = GenerationStamp.LAST_RESERVED_STAMP;
- DatanodeDescriptor dn = DFSTestUtil.getLocalDatanodeDescriptor();
- DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
- dn.getDatanodeUuid(), InetAddress.getLocalHost().getHostAddress());
- dn.isAlive = true;
-
- BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
- new Block(0, 1, oldGenstamp), (short) 1,
- HdfsServerConstants.BlockUCState.BEING_TRUNCATED,
- new DatanodeStorageInfo[] {storage});
+ INodesInPath iip = fsn.getFSDirectory().getINodesInPath4Write(src, true);
+ INodeFile file = iip.getLastINode().asFile();
+ long initialGenStamp = file.getLastBlock().getGenerationStamp();
+ // Test that prepareFileForTruncate sets up in-place truncate.
+ fsn.writeLock();
+ try {
+ Block oldBlock = file.getLastBlock();
+ Block truncateBlock =
+ fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ // In-place truncate uses old block id with new genStamp.
+ assertThat(truncateBlock.getBlockId(),
+ is(equalTo(oldBlock.getBlockId())));
+ assertThat(truncateBlock.getNumBytes(),
+ is(oldBlock.getNumBytes()));
+ assertThat(truncateBlock.getGenerationStamp(),
+ is(fsn.getBlockIdManager().getGenerationStampV2()));
+ assertThat(file.getLastBlock().getBlockUCState(),
+ is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
+ long blockRecoveryId = ((BlockInfoUnderConstruction) file.getLastBlock())
+ .getBlockRecoveryId();
+ assertThat(blockRecoveryId, is(initialGenStamp + 1));
+ fsn.getEditLog().logTruncate(
+ src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock);
+ } finally {
+ fsn.writeUnlock();
+ }
- inode.setBlocks(new BlockInfo[] {blockInfo});
+ // Re-create file and ensure we are ready to copy on truncate
+ writeContents(contents, BLOCK_SIZE, srcPath);
+ fs.allowSnapshot(parent);
+ fs.createSnapshot(parent, "ss0");
+ iip = fsn.getFSDirectory().getINodesInPath(src, true);
+ file = iip.getLastINode().asFile();
+ file.recordModification(iip.getLatestSnapshotId(), true);
+ assertThat(file.isBlockInLatestSnapshot(file.getLastBlock()), is(true));
+ initialGenStamp = file.getLastBlock().getGenerationStamp();
+ // Test that prepareFileForTruncate sets up copy-on-write truncate
fsn.writeLock();
try {
- fsn.initializeBlockRecovery(inode);
- assertThat(inode.getLastBlock().getBlockUCState(),
- is(HdfsServerConstants.BlockUCState.BEING_TRUNCATED));
- long blockRecoveryId = ((BlockInfoUnderConstruction) inode.getLastBlock())
+ Block oldBlock = file.getLastBlock();
+ Block truncateBlock =
+ fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ // Copy-on-write truncate makes new block with new id and genStamp
+ assertThat(truncateBlock.getBlockId(),
+ is(not(equalTo(oldBlock.getBlockId()))));
+ assertThat(truncateBlock.getNumBytes() < oldBlock.getNumBytes(),
+ is(true));
+ assertThat(truncateBlock.getGenerationStamp(),
+ is(fsn.getBlockIdManager().getGenerationStampV2()));
+ assertThat(file.getLastBlock().getBlockUCState(),
+ is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
+ long blockRecoveryId = ((BlockInfoUnderConstruction) file.getLastBlock())
.getBlockRecoveryId();
- assertThat(blockRecoveryId, is(oldGenstamp + 2));
+ assertThat(blockRecoveryId, is(initialGenStamp + 1));
+ fsn.getEditLog().logTruncate(
+ src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock);
} finally {
fsn.writeUnlock();
}
+ checkBlockRecovery(srcPath);
+ fs.deleteSnapshot(parent, "ss0");
+ fs.delete(parent, true);
}
static void writeContents(byte[] contents, int fileLength, Path p)
@@ -286,4 +703,38 @@ public class TestFileTruncate {
static LocatedBlocks getLocatedBlocks(Path src) throws IOException {
return fs.getClient().getLocatedBlocks(src.toString(), 0, Long.MAX_VALUE);
}
+
+ static void assertBlockExists(Block blk) {
+ assertNotNull("BlocksMap does not contain block: " + blk,
+ cluster.getNamesystem().getStoredBlock(blk));
+ }
+
+ static void assertBlockNotPresent(Block blk) {
+ assertNull("BlocksMap should not contain block: " + blk,
+ cluster.getNamesystem().getStoredBlock(blk));
+ }
+
+ static void assertFileLength(Path file, long length) throws IOException {
+ byte[] data = DFSTestUtil.readFileBuffer(fs, file);
+ assertEquals("Wrong data size in snapshot.", length, data.length);
+ }
+
+ static void checkFullFile(Path p, int newLength, byte[] contents)
+ throws IOException {
+ AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ }
+
+ static void restartCluster(StartupOption o)
+ throws IOException {
+ cluster.shutdown();
+ if(StartupOption.ROLLBACK == o)
+ NameNode.doRollback(conf, false);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM)
+ .format(false)
+ .nameNodePort(NameNode.DEFAULT_PORT)
+ .startupOption(o==StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
+ .dnStartupOption(o!=StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
+ .build();
+ fs = cluster.getFileSystem();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index 002c506..dce3f47 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9953e921/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index f61c075..6e8078b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<EDITS>
- <EDITS_VERSION>-60</EDITS_VERSION>
+ <EDITS_VERSION>-61</EDITS_VERSION>
<RECORD>
<OPCODE>OP_START_LOG_SEGMENT</OPCODE>
<DATA>
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1421822547136</EXPIRY_DATE>
- <KEY>24319c7d1f7c0828</KEY>
+ <EXPIRY_DATE>1421826999207</EXPIRY_DATE>
+ <KEY>ca9a0c8b240570b3</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1421822547140</EXPIRY_DATE>
- <KEY>254b1207021431f4</KEY>
+ <EXPIRY_DATE>1421826999210</EXPIRY_DATE>
+ <KEY>833c25a6fb2b0a6f</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,19 +37,19 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348286</MTIME>
- <ATIME>1421131348286</ATIME>
+ <MTIME>1421135800328</MTIME>
+ <ATIME>1421135800328</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>6</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>9</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -60,14 +60,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348328</MTIME>
- <ATIME>1421131348286</ATIME>
+ <MTIME>1421135800357</MTIME>
+ <ATIME>1421135800328</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -88,9 +88,9 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1421131348343</TIMESTAMP>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>9</RPC_CALLID>
+ <TIMESTAMP>1421135800368</TIMESTAMP>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>12</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -99,9 +99,9 @@
<TXID>8</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1421131348353</TIMESTAMP>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>10</RPC_CALLID>
+ <TIMESTAMP>1421135800377</TIMESTAMP>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>13</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -111,9 +111,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1421131348366</TIMESTAMP>
+ <TIMESTAMP>1421135800394</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -146,8 +146,8 @@
<TXID>13</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>15</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>18</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -157,8 +157,8 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>16</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>19</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -167,8 +167,8 @@
<TXID>15</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>17</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>20</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -179,19 +179,19 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348401</MTIME>
- <ATIME>1421131348401</ATIME>
+ <MTIME>1421135800442</MTIME>
+ <ATIME>1421135800442</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>18</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>21</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -202,14 +202,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348405</MTIME>
- <ATIME>1421131348401</ATIME>
+ <MTIME>1421135800445</MTIME>
+ <ATIME>1421135800442</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -265,10 +265,10 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1421131348436</TIMESTAMP>
+ <TIMESTAMP>1421135800485</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>25</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>28</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -279,19 +279,19 @@
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348443</MTIME>
- <ATIME>1421131348443</ATIME>
+ <MTIME>1421135800495</MTIME>
+ <ATIME>1421135800495</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>27</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>30</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -396,8 +396,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131348998</MTIME>
- <ATIME>1421131348443</ATIME>
+ <MTIME>1421135801050</MTIME>
+ <ATIME>1421135800495</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -418,7 +418,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -432,19 +432,19 @@
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349001</MTIME>
- <ATIME>1421131349001</ATIME>
+ <MTIME>1421135801053</MTIME>
+ <ATIME>1421135801053</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>38</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>41</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -549,8 +549,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349032</MTIME>
- <ATIME>1421131349001</ATIME>
+ <MTIME>1421135801091</MTIME>
+ <ATIME>1421135801053</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -571,7 +571,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -585,19 +585,19 @@
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349036</MTIME>
- <ATIME>1421131349036</ATIME>
+ <MTIME>1421135801095</MTIME>
+ <ATIME>1421135801095</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>47</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>50</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -702,8 +702,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349060</MTIME>
- <ATIME>1421131349036</ATIME>
+ <MTIME>1421135801126</MTIME>
+ <ATIME>1421135801095</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -724,7 +724,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -736,13 +736,13 @@
<TXID>57</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1421131349064</TIMESTAMP>
+ <TIMESTAMP>1421135801130</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>55</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>58</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -753,19 +753,19 @@
<INODEID>16392</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349068</MTIME>
- <ATIME>1421131349068</ATIME>
+ <MTIME>1421135810102</MTIME>
+ <ATIME>1421135810102</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>57</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>63</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -837,8 +837,8 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349085</MTIME>
- <ATIME>1421131349068</ATIME>
+ <MTIME>1421135810122</MTIME>
+ <ATIME>1421135810102</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -854,7 +854,7 @@
<GENSTAMP>1011</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -865,10 +865,10 @@
<DATA>
<TXID>66</TXID>
<SRC>/file_create</SRC>
- <CLIENTNAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENTNAME>
+ <CLIENTNAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENTNAME>
<CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
<NEWLENGTH>512</NEWLENGTH>
- <TIMESTAMP>1421131349088</TIMESTAMP>
+ <TIMESTAMP>1421135810125</TIMESTAMP>
</DATA>
</RECORD>
<RECORD>
@@ -879,15 +879,15 @@
<INODEID>16393</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1421131349095</MTIME>
- <ATIME>1421131349095</ATIME>
+ <MTIME>1421135810132</MTIME>
+ <ATIME>1421135810132</ATIME>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>64</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>70</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -898,19 +898,19 @@
<INODEID>16394</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131349098</MTIME>
- <ATIME>1421131349098</ATIME>
+ <MTIME>1421135810135</MTIME>
+ <ATIME>1421135810135</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>65</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>71</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -966,7 +966,7 @@
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
<TXID>74</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_526346936_1</LEASEHOLDER>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_240777107_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@@ -979,8 +979,8 @@
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421131351230</MTIME>
- <ATIME>1421131349098</ATIME>
+ <MTIME>1421135812235</MTIME>
+ <ATIME>1421135810135</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -991,7 +991,7 @@
<GENSTAMP>1013</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>plamenjeliazkov</USERNAME>
+ <USERNAME>shv</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -1002,13 +1002,13 @@
<DATA>
<TXID>76</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>plamenjeliazkov</OWNERNAME>
- <GROUPNAME>staff</GROUPNAME>
+ <OWNERNAME>shv</OWNERNAME>
+ <GROUPNAME>shv</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>72</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>78</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1017,8 +1017,8 @@
<TXID>77</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>73</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>79</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1029,9 +1029,9 @@
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844430345046085</EXPIRATION>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>74</RPC_CALLID>
+ <EXPIRATION>2305844430349507141</EXPIRATION>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1040,8 +1040,8 @@
<TXID>79</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>75</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1049,8 +1049,8 @@
<DATA>
<TXID>80</TXID>
<ID>1</ID>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>76</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>82</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1058,8 +1058,8 @@
<DATA>
<TXID>81</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>77</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>83</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1105,8 +1105,8 @@
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>79</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>85</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1119,8 +1119,8 @@
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>80</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>86</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1132,22 +1132,22 @@
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
</XATTR>
- <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
- <RPC_CALLID>81</RPC_CALLID>
+ <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID>
+ <RPC_CALLID>87</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
<TXID>86</TXID>
- <STARTTIME>1421131352186</STARTTIME>
+ <STARTTIME>1421135813268</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
<TXID>87</TXID>
- <FINALIZETIME>1421131352186</FINALIZETIME>
+ <FINALIZETIME>1421135813268</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
[02/14] hadoop git commit: YARN-3027. Scheduler should use
totalAvailable resource from node instead of availableResource for
maxAllocation. (adhoot via rkanter)
Posted by zh...@apache.org.
YARN-3027. Scheduler should use totalAvailable resource from node instead of availableResource for maxAllocation. (adhoot via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e29d9bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e29d9bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e29d9bc
Branch: refs/heads/HDFS-EC
Commit: 3e29d9bc83284fa1350ded0423f678f83f47c570
Parents: 82c57ef
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Jan 12 10:47:52 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:16 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/AbstractYarnScheduler.java | 12 +--
.../scheduler/TestAbstractYarnScheduler.java | 81 +++++++++++++++++++-
3 files changed, 89 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e29d9bc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 99c8514..153760c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -347,6 +347,9 @@ Release 2.7.0 - UNRELEASED
YARN-3014. Replaces labels on a host should update all NM's labels on that
host. (Wangda Tan via jianhe)
+ YARN-3027. Scheduler should use totalAvailable resource from node instead of
+ availableResource for maxAllocation. (adhoot via rkanter)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e29d9bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ad2b5d4..5d4d7e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -576,23 +576,23 @@ public abstract class AbstractYarnScheduler
writeLock.lock();
try {
if (add) { // added node
- int nodeMemory = node.getAvailableResource().getMemory();
+ int nodeMemory = node.getTotalResource().getMemory();
if (nodeMemory > maxNodeMemory) {
maxNodeMemory = nodeMemory;
maximumAllocation.setMemory(Math.min(
configuredMaximumAllocation.getMemory(), maxNodeMemory));
}
- int nodeVCores = node.getAvailableResource().getVirtualCores();
+ int nodeVCores = node.getTotalResource().getVirtualCores();
if (nodeVCores > maxNodeVCores) {
maxNodeVCores = nodeVCores;
maximumAllocation.setVirtualCores(Math.min(
configuredMaximumAllocation.getVirtualCores(), maxNodeVCores));
}
} else { // removed node
- if (maxNodeMemory == node.getAvailableResource().getMemory()) {
+ if (maxNodeMemory == node.getTotalResource().getMemory()) {
maxNodeMemory = -1;
}
- if (maxNodeVCores == node.getAvailableResource().getVirtualCores()) {
+ if (maxNodeVCores == node.getTotalResource().getVirtualCores()) {
maxNodeVCores = -1;
}
// We only have to iterate through the nodes if the current max memory
@@ -600,12 +600,12 @@ public abstract class AbstractYarnScheduler
if (maxNodeMemory == -1 || maxNodeVCores == -1) {
for (Map.Entry<NodeId, N> nodeEntry : nodes.entrySet()) {
int nodeMemory =
- nodeEntry.getValue().getAvailableResource().getMemory();
+ nodeEntry.getValue().getTotalResource().getMemory();
if (nodeMemory > maxNodeMemory) {
maxNodeMemory = nodeMemory;
}
int nodeVCores =
- nodeEntry.getValue().getAvailableResource().getVirtualCores();
+ nodeEntry.getValue().getTotalResource().getVirtualCores();
if (nodeVCores > maxNodeVCores) {
maxNodeVCores = nodeVCores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e29d9bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 67bdae2..27b20d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -25,11 +27,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestB
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
-
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("unchecked")
public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
public TestAbstractYarnScheduler(SchedulerType type) {
@@ -210,4 +218,75 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
Assert.assertEquals(0, scheduler.getNumClusterNodes());
}
+ @Test
+ public void testUpdateMaxAllocationUsesTotal() throws IOException {
+ final int configuredMaxVCores = 20;
+ final int configuredMaxMemory = 10 * 1024;
+ Resource configuredMaximumResource = Resource.newInstance
+ (configuredMaxMemory, configuredMaxVCores);
+
+ configureScheduler();
+ YarnConfiguration conf = getConf();
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ configuredMaxVCores);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ configuredMaxMemory);
+ conf.setLong(
+ YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
+ 0);
+
+ MockRM rm = new MockRM(conf);
+ try {
+ rm.start();
+ AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
+ .getResourceScheduler();
+
+ Resource emptyResource = Resource.newInstance(0, 0);
+ Resource fullResource1 = Resource.newInstance(1024, 5);
+ Resource fullResource2 = Resource.newInstance(2048, 10);
+
+ SchedulerNode mockNode1 = mock(SchedulerNode.class);
+ when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
+ when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
+ when(mockNode1.getTotalResource()).thenReturn(fullResource1);
+
+ SchedulerNode mockNode2 = mock(SchedulerNode.class);
+ when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
+ when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
+ when(mockNode2.getTotalResource()).thenReturn(fullResource2);
+
+ verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
+
+ scheduler.nodes = new HashMap<NodeId, SchedulerNode>();
+
+ scheduler.nodes.put(mockNode1.getNodeID(), mockNode1);
+ scheduler.updateMaximumAllocation(mockNode1, true);
+ verifyMaximumResourceCapability(fullResource1, scheduler);
+
+ scheduler.nodes.put(mockNode2.getNodeID(), mockNode2);
+ scheduler.updateMaximumAllocation(mockNode2, true);
+ verifyMaximumResourceCapability(fullResource2, scheduler);
+
+ scheduler.nodes.remove(mockNode2.getNodeID());
+ scheduler.updateMaximumAllocation(mockNode2, false);
+ verifyMaximumResourceCapability(fullResource1, scheduler);
+
+ scheduler.nodes.remove(mockNode1.getNodeID());
+ scheduler.updateMaximumAllocation(mockNode1, false);
+ verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
+ } finally {
+ rm.stop();
+ }
+ }
+
+ private void verifyMaximumResourceCapability(
+ Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
+
+ final Resource schedulerMaximumResourceCapability = scheduler
+ .getMaximumResourceCapability();
+ Assert.assertEquals(expectedMaximumResource.getMemory(),
+ schedulerMaximumResourceCapability.getMemory());
+ Assert.assertEquals(expectedMaximumResource.getVirtualCores(),
+ schedulerMaximumResourceCapability.getVirtualCores());
+ }
}
[04/14] hadoop git commit: HDFS-7598. Remove dependency on old
version of Guava in TestDFSClientCache#testEviction (Sangjin Lee via Colin P.
McCabe)
Posted by zh...@apache.org.
HDFS-7598. Remove dependency on old version of Guava in TestDFSClientCache#testEviction (Sangjin Lee via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db2abfc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db2abfc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db2abfc4
Branch: refs/heads/HDFS-EC
Commit: db2abfc4e6a924ee6261d85f979891bb8e92017d
Parents: 565a494
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jan 12 17:07:18 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java | 5 +++--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2abfc4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
index 6536cd8..54b7ee7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
@@ -39,7 +39,7 @@ public class TestDFSClientCache {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost");
// Only one entry will be in the cache
- final int MAX_CACHE_SIZE = 2;
+ final int MAX_CACHE_SIZE = 1;
DFSClientCache cache = new DFSClientCache(conf, MAX_CACHE_SIZE);
@@ -50,7 +50,8 @@ public class TestDFSClientCache {
cache.getDfsClient("test2");
assertTrue(isDfsClientClose(c1));
- assertEquals(MAX_CACHE_SIZE - 1, cache.clientCache.size());
+ assertTrue("cache size should be the max size or less",
+ cache.clientCache.size() <= MAX_CACHE_SIZE);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2abfc4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 05cb745..b57503a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -493,6 +493,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7326: Add documentation for hdfs debug commands (Vijay Bhat via Colin
P. McCabe)
+ HDFS-7598. Remove dependency on old version of guava in
+ TestDFSClientCache#testEviction. (Sangjin Lee via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
[14/14] hadoop git commit: HDFS-3107. Introduce truncate. Contributed
by Plamen Jeliazkov.
Posted by zh...@apache.org.
HDFS-3107. Introduce truncate. Contributed by Plamen Jeliazkov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/215c2af6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/215c2af6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/215c2af6
Branch: refs/heads/HDFS-EC
Commit: 215c2af6e167a2fc3ad993e9a70cf87a662a4146
Parents: b61ee33
Author: Plamen Jeliazkov <pl...@gmail.com>
Authored: Mon Jan 12 21:53:52 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:18 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 15 +
.../hadoop/hdfs/DistributedFileSystem.java | 15 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 32 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +
.../ClientNamenodeProtocolTranslatorPB.java | 16 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +-
.../BlockInfoUnderConstruction.java | 6 +-
.../server/blockmanagement/DatanodeManager.java | 5 +-
.../blockmanagement/DatanodeStorageInfo.java | 2 +-
.../hdfs/server/common/HdfsServerConstants.java | 7 +
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +-
.../hdfs/server/namenode/FSDirectory.java | 66 ++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 15 +
.../hdfs/server/namenode/FSEditLogLoader.java | 7 +
.../hdfs/server/namenode/FSEditLogOp.java | 111 +++++
.../hdfs/server/namenode/FSEditLogOpCodes.java | 1 +
.../hdfs/server/namenode/FSNamesystem.java | 116 +++++-
.../hadoop/hdfs/server/namenode/INodeFile.java | 39 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 17 +
.../namenode/metrics/NameNodeMetrics.java | 5 +
.../snapshot/FileWithSnapshotFeature.java | 37 +-
.../server/protocol/BlockRecoveryCommand.java | 17 +
.../src/main/proto/ClientNamenodeProtocol.proto | 11 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 1 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 8 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../hdfs/server/namenode/TestFileTruncate.java | 289 +++++++++++++
.../server/namenode/TestNamenodeRetryCache.java | 4 +-
.../namenode/ha/TestRetryCacheWithHA.java | 4 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5065 -> 5791 bytes
.../src/test/resources/editsStored.xml | 404 ++++++++++++-------
32 files changed, 1091 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cefdc16..6da0667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -18,6 +18,8 @@ Trunk (Unreleased)
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
+ HDFS-3107. Introduce truncate. (Plamen Jeliazkov via shv)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 62db1fa..f289da7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1916,6 +1916,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
SnapshotAccessControlException.class);
}
}
+
+ /**
+ * Truncate a file to an indicated size
+ * See {@link ClientProtocol#truncate(String, long)}.
+ */
+ public boolean truncate(String src, long newLength) throws IOException {
+ checkOpen();
+ try {
+ return namenode.truncate(src, newLength, clientName);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ UnresolvedPathException.class);
+ }
+ }
+
/**
* Delete file or directory.
* See {@link ClientProtocol#delete(String, boolean)}.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index d4653ac..6284f61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -626,7 +626,20 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absDst);
}
}
-
+
+ /**
+ * Truncate the file in the indicated path to the indicated size.
+ * @param f The path to the file to be truncated
+ * @param newLength The size the file is to be truncated to
+ *
+ * @return true if and client does not need to wait for block recovery,
+ * false if client needs to wait for block recovery.
+ */
+ public boolean truncate(Path f, final long newLength) throws IOException {
+ statistics.incrementWriteOps(1);
+ return dfs.truncate(getPathName(f), newLength);
+ }
+
@Override
public boolean delete(Path f, final boolean recursive) throws IOException {
statistics.incrementWriteOps(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 2301575..749f387 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -521,7 +521,37 @@ public interface ClientProtocol {
FileAlreadyExistsException, FileNotFoundException,
NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
UnresolvedLinkException, SnapshotAccessControlException, IOException;
-
+
+ /**
+ * Truncate file src to new size.
+ * <ul>
+ * <li>Fails if src is a directory.
+ * <li>Fails if src does not exist.
+ * <li>Fails if src is not closed.
+ * <li>Fails if new size is greater than current size.
+ * </ul>
+ * <p>
+ * This implementation of truncate is purely a namespace operation if truncate
+ * occurs at a block boundary. Requires DataNode block recovery otherwise.
+ * <p>
+ * @param src existing file
+ * @param newLength the target size
+ *
+ * @return true if and client does not need to wait for block recovery,
+ * false if client needs to wait for block recovery.
+ *
+ * @throws AccessControlException If access is denied
+ * @throws FileNotFoundException If file <code>src</code> is not found
+ * @throws SafeModeException truncate not allowed in safemode
+ * @throws UnresolvedLinkException If <code>src</code> contains a symlink
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ * @throws IOException If an I/O error occurred
+ */
+ @Idempotent
+ public boolean truncate(String src, long newLength, String clientName)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, SnapshotAccessControlException, IOException;
+
/**
* Delete the given file or directory from the file system.
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 5b6609b..8bcc1eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -181,6 +181,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetSto
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
@@ -585,6 +587,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public TruncateResponseProto truncate(RpcController controller,
+ TruncateRequestProto req) throws ServiceException {
+ try {
+ boolean result = server.truncate(req.getSrc(), req.getNewLength(),
+ req.getClientName());
+ return TruncateResponseProto.newBuilder().setResult(result).build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public DeleteResponseProto delete(RpcController controller,
DeleteRequestProto req) throws ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 5804920..f3826af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -155,6 +155,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetQuo
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetReplicationRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetSafeModeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
@@ -302,6 +303,21 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public boolean truncate(String src, long newLength, String clientName)
+ throws IOException, UnresolvedLinkException {
+ TruncateRequestProto req = TruncateRequestProto.newBuilder()
+ .setSrc(src)
+ .setNewLength(newLength)
+ .setClientName(clientName)
+ .build();
+ try {
+ return rpcProxy.truncate(null, req).getResult();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public LastBlockWithStatus append(String src, String clientName)
throws AccessControlException, DSQuotaExceededException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index ee6d58c..3f6a7f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -608,13 +608,15 @@ public class PBHelper {
}
LocatedBlockProto lb = PBHelper.convert((LocatedBlock)b);
return RecoveringBlockProto.newBuilder().setBlock(lb)
- .setNewGenStamp(b.getNewGenerationStamp()).build();
+ .setNewGenStamp(b.getNewGenerationStamp())
+ .setTruncateFlag(b.getTruncateFlag()).build();
}
public static RecoveringBlock convert(RecoveringBlockProto b) {
ExtendedBlock block = convert(b.getBlock().getB());
DatanodeInfo[] locs = convert(b.getBlock().getLocsList());
- return new RecoveringBlock(block, locs, b.getNewGenStamp());
+ return new RecoveringBlock(block, locs, b.getNewGenStamp(),
+ b.getTruncateFlag());
}
public static DatanodeInfoProto.AdminState convert(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
index f19ad1c..28b179d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -273,7 +273,11 @@ public class BlockInfoUnderConstruction extends BlockInfo {
* make it primary.
*/
public void initializeBlockRecovery(long recoveryId) {
- setBlockUCState(BlockUCState.UNDER_RECOVERY);
+ initializeBlockRecovery(BlockUCState.UNDER_RECOVERY, recoveryId);
+ }
+
+ public void initializeBlockRecovery(BlockUCState s, long recoveryId) {
+ setBlockUCState(s);
blockRecoveryId = recoveryId;
if (replicas.size() == 0) {
NameNode.blockStateChangeLog.warn("BLOCK*"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 41d0363..918b8d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -1439,10 +1440,12 @@ public class DatanodeManager {
LOG.info("Skipped stale nodes for recovery : " +
(storages.length - recoveryLocations.size()));
}
+ boolean isTruncate = b.getBlockUCState().equals(
+ HdfsServerConstants.BlockUCState.BEING_TRUNCATED);
brCommand.add(new RecoveringBlock(
new ExtendedBlock(blockPoolId, b),
DatanodeStorageInfo.toDatanodeInfos(recoveryLocations),
- b.getBlockRecoveryId()));
+ b.getBlockRecoveryId(), isTruncate));
} else {
// If too many replicas are stale, then choose all replicas to participate
// in block recovery.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index a3198e2..3ab10b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -240,7 +240,7 @@ public class DatanodeStorageInfo {
return result;
}
- boolean removeBlock(BlockInfo b) {
+ public boolean removeBlock(BlockInfo b) {
blockList = b.listRemove(blockList, this);
if (b.removeStorage(this)) {
numBlocks--;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 9bba2c9..f2e7ff4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -300,6 +300,13 @@ public final class HdfsServerConstants {
*/
UNDER_RECOVERY,
/**
+ * The block is being truncated.<br>
+ * When a file is truncated its last block may need to be truncated
+ * and needs to go through a recovery procedure,
+ * which synchronizes the existing replicas contents.
+ */
+ BEING_TRUNCATED,
+ /**
* The block is committed.<br>
* The client reported that all bytes are written to data-nodes
* with the given generation stamp and block length, but no
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index d5d8429..7f95f33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2691,7 +2691,10 @@ public class DataNode extends ReconfigurableBase
r.rInfo.getNumBytes() == finalizedLength)
participatingList.add(r);
}
- newBlock.setNumBytes(finalizedLength);
+ if(rBlock.getTruncateFlag())
+ newBlock.setNumBytes(rBlock.getBlock().getNumBytes());
+ else
+ newBlock.setNumBytes(finalizedLength);
break;
case RBW:
case RWR:
@@ -2703,7 +2706,10 @@ public class DataNode extends ReconfigurableBase
participatingList.add(r);
}
}
- newBlock.setNumBytes(minLength);
+ if(rBlock.getTruncateFlag())
+ newBlock.setNumBytes(rBlock.getBlock().getNumBytes());
+ else
+ newBlock.setNumBytes(minLength);
break;
case RUR:
case TEMPORARY:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b39519f..1948099 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1087,7 +1087,71 @@ public class FSDirectory implements Closeable {
public INodeMap getINodeMap() {
return inodeMap;
}
-
+
+ /**
+ * FSEditLogLoader implementation.
+ * Unlike FSNamesystem.truncate, this will not schedule block recovery.
+ */
+ void unprotectedTruncate(String src, String clientName, String clientMachine,
+ long newLength, long mtime)
+ throws UnresolvedLinkException, QuotaExceededException,
+ SnapshotAccessControlException, IOException {
+ INodesInPath iip = getINodesInPath(src, true);
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ boolean onBlockBoundary =
+ unprotectedTruncate(iip, newLength, collectedBlocks, mtime);
+
+ if(! onBlockBoundary) {
+ getFSNamesystem().prepareFileForWrite(src,
+ iip, clientName, clientMachine, false, false);
+ }
+ getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+ }
+
+ boolean truncate(INodesInPath iip, long newLength,
+ BlocksMapUpdateInfo collectedBlocks,
+ long mtime)
+ throws IOException {
+ writeLock();
+ try {
+ return unprotectedTruncate(iip, newLength, collectedBlocks, mtime);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Truncate has the following properties:
+ * 1.) Any block deletions occur now.
+ * 2.) INode length is truncated now – clients can only read up to new length.
+ * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
+ * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
+ * 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
+ * 6.) Soft and hard Lease expiration require truncation recovery to complete.
+ *
+ * @return true if on the block boundary or false if recovery is need
+ */
+ boolean unprotectedTruncate(INodesInPath iip, long newLength,
+ BlocksMapUpdateInfo collectedBlocks,
+ long mtime) throws IOException {
+ assert hasWriteLock();
+ INodeFile file = iip.getLastINode().asFile();
+ long oldDiskspace = file.diskspaceConsumed();
+ long remainingLength =
+ file.collectBlocksBeyondMax(newLength, collectedBlocks);
+ file.setModificationTime(mtime);
+ updateCount(iip, 0, file.diskspaceConsumed() - oldDiskspace, true);
+ // If on block boundary, then return
+ long lastBlockDelta = remainingLength - newLength;
+ if(lastBlockDelta == 0)
+ return true;
+ // Set new last block length
+ BlockInfo lastBlock = file.getLastBlock();
+ assert lastBlock.getNumBytes() - lastBlockDelta > 0 : "wrong block size";
+ lastBlock.setNumBytes(lastBlock.getNumBytes() - lastBlockDelta);
+ return false;
+ }
+
/**
* This method is always called with writeLock of FSDirectory held.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 4a29b59..d32aad9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetStoragePolicyOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetXAttrOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TruncateOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeOp;
@@ -896,6 +897,20 @@ public class FSEditLog implements LogsPurgeable {
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
+
+ /**
+ * Add truncate file record to edit log
+ */
+ void logTruncate(String src, String clientName, String clientMachine,
+ long size, long timestamp) {
+ TruncateOp op = TruncateOp.getInstance(cache.get())
+ .setPath(src)
+ .setClientName(clientName)
+ .setClientMachine(clientMachine)
+ .setNewLength(size)
+ .setTimestamp(timestamp);
+ logEdit(op);
+ }
/**
* Add legacy block generation stamp record to edit log
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 0118926..2ff3b77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TruncateOp;
import static org.apache.hadoop.hdfs.server.namenode.FSImageFormat.renameReservedPathsOnUpgrade;
import static org.apache.hadoop.util.Time.now;
@@ -853,6 +854,12 @@ public class FSEditLogLoader {
}
break;
}
+ case OP_TRUNCATE: {
+ TruncateOp truncateOp = (TruncateOp) op;
+ fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
+ truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp);
+ break;
+ }
case OP_SET_STORAGE_POLICY: {
SetStoragePolicyOp setStoragePolicyOp = (SetStoragePolicyOp) op;
final String path = renameReservedPathsOnUpgrade(setStoragePolicyOp.path,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 11026fc..396fb08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -59,6 +59,7 @@ import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SET_XAT
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_START_LOG_SEGMENT;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SYMLINK;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_TIMES;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_TRUNCATE;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_BLOCKS;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_MASTER_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SET_STORAGE_POLICY;
@@ -180,6 +181,7 @@ public abstract class FSEditLogOp {
inst.put(OP_START_LOG_SEGMENT, new LogSegmentOp(OP_START_LOG_SEGMENT));
inst.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT));
inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
+ inst.put(OP_TRUNCATE, new TruncateOp());
inst.put(OP_ALLOW_SNAPSHOT, new AllowSnapshotOp());
inst.put(OP_DISALLOW_SNAPSHOT, new DisallowSnapshotOp());
@@ -2602,6 +2604,115 @@ public abstract class FSEditLogOp {
readRpcIdsFromXml(st);
}
}
+
+ static class TruncateOp extends FSEditLogOp {
+ String src;
+ String clientName;
+ String clientMachine;
+ long newLength;
+ long timestamp;
+
+ private TruncateOp() {
+ super(OP_TRUNCATE);
+ }
+
+ static TruncateOp getInstance(OpInstanceCache cache) {
+ return (TruncateOp)cache.get(OP_TRUNCATE);
+ }
+
+ @Override
+ void resetSubFields() {
+ src = null;
+ clientName = null;
+ clientMachine = null;
+ newLength = 0L;
+ timestamp = 0L;
+ }
+
+ TruncateOp setPath(String src) {
+ this.src = src;
+ return this;
+ }
+
+ TruncateOp setClientName(String clientName) {
+ this.clientName = clientName;
+ return this;
+ }
+
+ TruncateOp setClientMachine(String clientMachine) {
+ this.clientMachine = clientMachine;
+ return this;
+ }
+
+ TruncateOp setNewLength(long newLength) {
+ this.newLength = newLength;
+ return this;
+ }
+
+ TruncateOp setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ @Override
+ void readFields(DataInputStream in, int logVersion) throws IOException {
+ src = FSImageSerialization.readString(in);
+ clientName = FSImageSerialization.readString(in);
+ clientMachine = FSImageSerialization.readString(in);
+ newLength = FSImageSerialization.readLong(in);
+ timestamp = FSImageSerialization.readLong(in);
+ }
+
+ @Override
+ public void writeFields(DataOutputStream out) throws IOException {
+ FSImageSerialization.writeString(src, out);
+ FSImageSerialization.writeString(clientName, out);
+ FSImageSerialization.writeString(clientMachine, out);
+ FSImageSerialization.writeLong(newLength, out);
+ FSImageSerialization.writeLong(timestamp, out);
+ }
+
+ @Override
+ protected void toXml(ContentHandler contentHandler) throws SAXException {
+ XMLUtils.addSaxString(contentHandler, "SRC", src);
+ XMLUtils.addSaxString(contentHandler, "CLIENTNAME", clientName);
+ XMLUtils.addSaxString(contentHandler, "CLIENTMACHINE", clientMachine);
+ XMLUtils.addSaxString(contentHandler, "NEWLENGTH",
+ Long.toString(newLength));
+ XMLUtils.addSaxString(contentHandler, "TIMESTAMP",
+ Long.toString(timestamp));
+ }
+
+ @Override
+ void fromXml(Stanza st) throws InvalidXmlException {
+ this.src = st.getValue("SRC");
+ this.clientName = st.getValue("CLIENTNAME");
+ this.clientMachine = st.getValue("CLIENTMACHINE");
+ this.newLength = Long.parseLong(st.getValue("NEWLENGTH"));
+ this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("TruncateOp [src=");
+ builder.append(src);
+ builder.append(", clientName=");
+ builder.append(clientName);
+ builder.append(", clientMachine=");
+ builder.append(clientMachine);
+ builder.append(", newLength=");
+ builder.append(newLength);
+ builder.append(", timestamp=");
+ builder.append(timestamp);
+ builder.append(", opCode=");
+ builder.append(opCode);
+ builder.append(", txid=");
+ builder.append(txid);
+ builder.append("]");
+ return builder.toString();
+ }
+ }
/**
* {@literal @Idempotent} for {@link ClientProtocol#recoverLease}. In the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
index 86be54a..468e048 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
@@ -73,6 +73,7 @@ public enum FSEditLogOpCodes {
OP_SET_XATTR ((byte) 43),
OP_REMOVE_XATTR ((byte) 44),
OP_SET_STORAGE_POLICY ((byte) 45),
+ OP_TRUNCATE ((byte) 46),
// Note that the current range of the valid OP code is 0~127
OP_INVALID ((byte) -1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 36a4334..c250838 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1907,6 +1907,114 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Truncate file to a lower length.
+ * Truncate cannot be reverted / recovered from as it causes data loss.
+ * Truncation at block boundary is atomic, otherwise it requires
+ * block recovery to truncate the last block of the file.
+ *
+ * @return true if and client does not need to wait for block recovery,
+ * false if client needs to wait for block recovery.
+ */
+ boolean truncate(String src, long newLength,
+ String clientName, String clientMachine,
+ long mtime)
+ throws IOException, UnresolvedLinkException {
+ boolean ret;
+ try {
+ ret = truncateInt(src, newLength, clientName, clientMachine, mtime);
+ } catch (AccessControlException e) {
+ logAuditEvent(false, "truncate", src);
+ throw e;
+ }
+ return ret;
+ }
+
+ boolean truncateInt(String srcArg, long newLength,
+ String clientName, String clientMachine,
+ long mtime)
+ throws IOException, UnresolvedLinkException {
+ String src = srcArg;
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.truncate: src="
+ + src + " newLength=" + newLength);
+ }
+ HdfsFileStatus stat = null;
+ FSPermissionChecker pc = getPermissionChecker();
+ checkOperation(OperationCategory.WRITE);
+ boolean res;
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot truncate for " + src);
+ src = dir.resolvePath(pc, src, pathComponents);
+ res = truncateInternal(src, newLength, clientName,
+ clientMachine, mtime, pc);
+ stat = FSDirStatAndListingOp.getFileInfo(dir, src, false,
+ FSDirectory.isReservedRawName(src), true);
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ logAuditEvent(true, "truncate", src, null, stat);
+ return res;
+ }
+
+ /**
+ * Truncate a file to a given size
+ * Update the count at each ancestor directory with quota
+ */
+ boolean truncateInternal(String src, long newLength,
+ String clientName, String clientMachine,
+ long mtime, FSPermissionChecker pc)
+ throws IOException, UnresolvedLinkException {
+ assert hasWriteLock();
+ INodesInPath iip = dir.getINodesInPath4Write(src, true);
+ if (isPermissionEnabled) {
+ dir.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ INodeFile file = iip.getLastINode().asFile();
+ // Data will be lost after truncate occurs so it cannot support snapshots.
+ if(file.isInLatestSnapshot(iip.getLatestSnapshotId()))
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate file with snapshot.");
+ // Opening an existing file for write. May need lease recovery.
+ recoverLeaseInternal(iip, src, clientName, clientMachine, false);
+ // Refresh INode as the file could have been closed
+ iip = dir.getINodesInPath4Write(src, true);
+ file = INodeFile.valueOf(iip.getLastINode(), src);
+ // Truncate length check.
+ long oldLength = file.computeFileSize();
+ if(oldLength == newLength)
+ return true;
+ if(oldLength < newLength)
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate to a larger file size. Current size: " + oldLength +
+ ", truncate size: " + newLength + ".");
+ // Perform INodeFile truncation.
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ boolean onBlockBoundary = dir.truncate(iip, newLength,
+ collectedBlocks, mtime);
+
+ if(! onBlockBoundary) {
+ // Open file for write, but don't log into edits
+ prepareFileForWrite(src, iip, clientName, clientMachine, false, false);
+ file = INodeFile.valueOf(dir.getINode4Write(src), src);
+ initializeBlockRecovery(file);
+ }
+ getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime);
+ removeBlocks(collectedBlocks);
+ return onBlockBoundary;
+ }
+
+ void initializeBlockRecovery(INodeFile inodeFile) throws IOException {
+ BlockInfo lastBlock = inodeFile.getLastBlock();
+ long recoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(lastBlock));
+ ((BlockInfoUnderConstruction)lastBlock).initializeBlockRecovery(
+ BlockUCState.BEING_TRUNCATED, recoveryId);
+ }
+
+ /**
* Create a symbolic link.
*/
void createSymlink(String target, String link,
@@ -2615,7 +2723,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} else {
final BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null
- && lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
+ && (lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY ||
+ lastBlock.getBlockUCState() == BlockUCState.BEING_TRUNCATED)) {
throw new RecoveryInProgressException("Recovery in progress, file ["
+ src + "], " + "lease owner [" + lease.getHolder() + "]");
} else {
@@ -3833,6 +3942,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new AlreadyBeingCreatedException(message);
case UNDER_CONSTRUCTION:
case UNDER_RECOVERY:
+ case BEING_TRUNCATED:
final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock;
// setup the last block locations from the blockManager if not known
if (uc.getNumExpectedLocations() == 0) {
@@ -3854,7 +3964,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// start recovery of the last block for this file
long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
- uc.initializeBlockRecovery(blockRecoveryId);
+ if (uc.getBlockUCState() != BlockUCState.BEING_TRUNCATED) {
+ uc.initializeBlockRecovery(blockRecoveryId);
+ }
leaseManager.renewLease(lease);
// Cannot close file right now, since the last block requires recovery.
// This may potentially cause infinite loop in lease recovery
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index b811f12..d1ff2f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -696,4 +696,43 @@ public class INodeFile extends INodeWithAdditionalFields
out.print(blocks == null || blocks.length == 0? null: blocks[0]);
out.println();
}
+
+ /**
+ * Remove full blocks at the end file up to newLength
+ * @return sum of sizes of the remained blocks
+ */
+ public long collectBlocksBeyondMax(final long max,
+ final BlocksMapUpdateInfo collectedBlocks) {
+ final BlockInfo[] oldBlocks = getBlocks();
+ if (oldBlocks == null)
+ return 0;
+ //find the minimum n such that the size of the first n blocks > max
+ int n = 0;
+ long size = 0;
+ for(; n < oldBlocks.length && max > size; n++) {
+ size += oldBlocks[n].getNumBytes();
+ }
+ if (n >= oldBlocks.length)
+ return size;
+
+ // starting from block n, the data is beyond max.
+ // resize the array.
+ final BlockInfo[] newBlocks;
+ if (n == 0) {
+ newBlocks = BlockInfo.EMPTY_ARRAY;
+ } else {
+ newBlocks = new BlockInfo[n];
+ System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
+ }
+ // set new blocks
+ setBlocks(newBlocks);
+
+ // collect the blocks beyond max
+ if (collectedBlocks != null) {
+ for(; n < oldBlocks.length; n++) {
+ collectedBlocks.addDeleteBlock(oldBlocks[n]);
+ }
+ }
+ return size;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 8240946..6ef8fd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_PATH_DEPTH;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_PATH_LENGTH;
+import static org.apache.hadoop.util.Time.now;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -883,6 +884,22 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
+ public boolean truncate(String src, long newLength, String clientName)
+ throws IOException {
+ if(stateChangeLog.isDebugEnabled()) {
+ stateChangeLog.debug("*DIR* NameNode.truncate: " + src + " to " +
+ newLength);
+ }
+ String clientMachine = getClientMachine();
+ try {
+ return namesystem.truncate(
+ src, newLength, clientName, clientMachine, now());
+ } finally {
+ metrics.incrFilesTruncated();
+ }
+ }
+
+ @Override // ClientProtocol
public boolean delete(String src, boolean recursive) throws IOException {
if (stateChangeLog.isDebugEnabled()) {
stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
index 42942dc..94e845b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
@@ -47,6 +47,7 @@ public class NameNodeMetrics {
@Metric MutableCounterLong filesAppended;
@Metric MutableCounterLong getBlockLocations;
@Metric MutableCounterLong filesRenamed;
+ @Metric MutableCounterLong filesTruncated;
@Metric MutableCounterLong getListingOps;
@Metric MutableCounterLong deleteFileOps;
@Metric("Number of files/dirs deleted by delete or rename operations")
@@ -173,6 +174,10 @@ public class NameNodeMetrics {
filesRenamed.incr();
}
+ public void incrFilesTruncated() {
+ filesTruncated.incr();
+ }
+
public void incrFilesDeleted(long delta) {
filesDeleted.incr(delta);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index e3bf349..16f534f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -182,40 +181,6 @@ public class FileWithSnapshotFeature implements INode.Feature {
max = file.computeFileSize();
}
- collectBlocksBeyondMax(file, max, info);
- }
-
- private void collectBlocksBeyondMax(final INodeFile file, final long max,
- final BlocksMapUpdateInfo collectedBlocks) {
- final BlockInfo[] oldBlocks = file.getBlocks();
- if (oldBlocks != null) {
- //find the minimum n such that the size of the first n blocks > max
- int n = 0;
- for(long size = 0; n < oldBlocks.length && max > size; n++) {
- size += oldBlocks[n].getNumBytes();
- }
-
- // starting from block n, the data is beyond max.
- if (n < oldBlocks.length) {
- // resize the array.
- final BlockInfo[] newBlocks;
- if (n == 0) {
- newBlocks = BlockInfo.EMPTY_ARRAY;
- } else {
- newBlocks = new BlockInfo[n];
- System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
- }
-
- // set new blocks
- file.setBlocks(newBlocks);
-
- // collect the blocks beyond max.
- if (collectedBlocks != null) {
- for(; n < oldBlocks.length; n++) {
- collectedBlocks.addDeleteBlock(oldBlocks[n]);
- }
- }
- }
- }
+ file.collectBlocksBeyondMax(max, info);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
index b7199ba..c512038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
@@ -53,6 +53,7 @@ public class BlockRecoveryCommand extends DatanodeCommand {
@InterfaceAudience.Private
@InterfaceStability.Evolving
public static class RecoveringBlock extends LocatedBlock {
+ private boolean truncate;
private final long newGenerationStamp;
/**
@@ -64,12 +65,28 @@ public class BlockRecoveryCommand extends DatanodeCommand {
}
/**
+ * RecoveryingBlock with truncate option.
+ */
+ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS,
+ boolean truncate) {
+ this(b, locs, newGS);
+ this.truncate = truncate;
+ }
+
+ /**
* Return the new generation stamp of the block,
* which also plays role of the recovery id.
*/
public long getNewGenerationStamp() {
return newGenerationStamp;
}
+
+ /**
+ * Return whether to truncate the block to the ExtendedBlock's length.
+ */
+ public boolean getTruncateFlag() {
+ return truncate;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
index 2c1d3cb..5c9f752 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
@@ -198,6 +198,16 @@ message ConcatRequestProto {
message ConcatResponseProto { // void response
}
+message TruncateRequestProto {
+ required string src = 1;
+ required uint64 newLength = 2;
+ required string clientName = 3;
+}
+
+message TruncateResponseProto {
+ required bool result = 1;
+}
+
message RenameRequestProto {
required string src = 1;
required string dst = 2;
@@ -722,6 +732,7 @@ service ClientNamenodeProtocol {
rpc reportBadBlocks(ReportBadBlocksRequestProto)
returns(ReportBadBlocksResponseProto);
rpc concat(ConcatRequestProto) returns(ConcatResponseProto);
+ rpc truncate(TruncateRequestProto) returns(TruncateResponseProto);
rpc rename(RenameRequestProto) returns(RenameResponseProto);
rpc rename2(Rename2RequestProto) returns(Rename2ResponseProto);
rpc delete(DeleteRequestProto) returns(DeleteResponseProto);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index 04a8f3f..d989c0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -556,6 +556,7 @@ enum ReplicaStateProto {
message RecoveringBlockProto {
required uint64 newGenStamp = 1; // New genstamp post recovery
required LocatedBlockProto block = 2; // Block to be recovered
+ optional bool truncateFlag = 3; // Block needs to be truncated
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 01f5d2e..15f5f2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1194,7 +1194,13 @@ public class DFSTestUtil {
DFSTestUtil.createFile(filesystem, pathConcatFiles[1], length, replication,
seed);
filesystem.concat(pathConcatTarget, pathConcatFiles);
-
+
+ // OP_TRUNCATE 46
+ length = blockSize * 2;
+ DFSTestUtil.createFile(filesystem, pathFileCreate, length, replication,
+ seed);
+ filesystem.truncate(pathFileCreate, blockSize);
+
// OP_SYMLINK 17
Path pathSymlink = new Path("/file_symlink");
fc.createSymlink(pathConcatTarget, pathSymlink, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 77a17ed..75a4ad4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -71,7 +71,7 @@ public class TestDFSInotifyEventInputStream {
*/
@Test
public void testOpcodeCount() {
- Assert.assertEquals(47, FSEditLogOpCodes.values().length);
+ Assert.assertEquals(48, FSEditLogOpCodes.values().length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
new file mode 100644
index 0000000..ba9d04e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.InetAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFileTruncate {
+ static {
+ GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
+ GenericTestUtils.setLogLevel(FSEditLogLoader.LOG, Level.ALL);
+ }
+ static final int BLOCK_SIZE = 4;
+ static final short REPLICATION = 3;
+ static final int DATANODE_NUM = 3;
+ static final int SUCCESS_ATTEMPTS = 300;
+ static final int RECOVERY_ATTEMPTS = 600;
+ static final long SLEEP = 100L;
+
+ static final long LOW_SOFTLIMIT = 100L;
+ static final long LOW_HARDLIMIT = 200L;
+ static final int SHORT_HEARTBEAT = 1;
+
+ static Configuration conf;
+ static MiniDFSCluster cluster;
+ static DistributedFileSystem fs;
+
+ @BeforeClass
+ public static void startUp() throws IOException {
+ conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, SHORT_HEARTBEAT);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .format(true)
+ .numDataNodes(DATANODE_NUM)
+ .nameNodePort(NameNode.DEFAULT_PORT)
+ .waitSafeMode(true)
+ .build();
+ fs = cluster.getFileSystem();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if(fs != null) fs.close();
+ if(cluster != null) cluster.shutdown();
+ }
+
+ /**
+ * Truncate files of different sizes byte by byte.
+ */
+ @Test
+ public void testBasicTruncate() throws IOException {
+ int startingFileSize = 3 * BLOCK_SIZE;
+
+ Path parent = new Path("/test");
+ fs.mkdirs(parent);
+ fs.setQuota(parent, 100, 1000);
+ byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+ for (int fileLength = startingFileSize; fileLength > 0;
+ fileLength -= BLOCK_SIZE - 1) {
+ for (int toTruncate = 0; toTruncate <= fileLength; toTruncate++) {
+ final Path p = new Path(parent, "testBasicTruncate" + fileLength);
+ writeContents(contents, fileLength, p);
+
+ int newLength = fileLength - toTruncate;
+ boolean isReady = fs.truncate(p, newLength);
+
+ if(!isReady)
+ checkBlockRecovery(p);
+
+ FileStatus fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLength));
+
+ ContentSummary cs = fs.getContentSummary(parent);
+ assertEquals("Bad disk space usage",
+ cs.getSpaceConsumed(), newLength * REPLICATION);
+ // validate the file content
+ AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ }
+ }
+ fs.delete(parent, true);
+ }
+
+ /**
+ * Failure / recovery test for truncate.
+ * In this failure the DNs fail to recover the blocks and the NN triggers
+ * lease recovery.
+ * File stays in RecoveryInProgress until DataNodes report recovery.
+ */
+ @Test
+ public void testTruncateFailure() throws IOException {
+ int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+ int toTruncate = 1;
+
+ byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+ final Path p = new Path("/testTruncateFailure");
+ FSDataOutputStream out = fs.create(p, false, BLOCK_SIZE, REPLICATION,
+ BLOCK_SIZE);
+ out.write(contents, 0, startingFileSize);
+ try {
+ fs.truncate(p, 0);
+ fail("Truncate must fail on open file.");
+ } catch(IOException expected) {}
+ out.close();
+
+ cluster.shutdownDataNodes();
+ NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
+ .setLeasePeriod(LOW_SOFTLIMIT, LOW_HARDLIMIT);
+
+ int newLength = startingFileSize - toTruncate;
+ boolean isReady = fs.truncate(p, newLength);
+ assertThat("truncate should have triggered block recovery.",
+ isReady, is(false));
+ FileStatus fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLength));
+
+ boolean recoveryTriggered = false;
+ for(int i = 0; i < RECOVERY_ATTEMPTS; i++) {
+ String leaseHolder =
+ NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(),
+ p.toUri().getPath());
+ if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
+ cluster.startDataNodes(conf, DATANODE_NUM, true,
+ HdfsServerConstants.StartupOption.REGULAR, null);
+ recoveryTriggered = true;
+ break;
+ }
+ try { Thread.sleep(SLEEP); } catch (InterruptedException ignored) {}
+ }
+ assertThat("lease recovery should have occurred in ~" +
+ SLEEP * RECOVERY_ATTEMPTS + " ms.", recoveryTriggered, is(true));
+
+ checkBlockRecovery(p);
+
+ NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
+ .setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
+ HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+
+ fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLength));
+
+ AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ fs.delete(p, false);
+ }
+
+ /**
+ * EditLogOp load test for Truncate.
+ */
+ @Test
+ public void testTruncateEditLogLoad() throws IOException {
+ int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+ int toTruncate = 1;
+
+ byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+
+ final Path p = new Path("/testTruncateEditLogLoad");
+ writeContents(contents, startingFileSize, p);
+
+ int newLength = startingFileSize - toTruncate;
+ boolean isReady = fs.truncate(p, newLength);
+ assertThat("truncate should have triggered block recovery.",
+ isReady, is(false));
+
+ checkBlockRecovery(p);
+
+ cluster.restartNameNode();
+
+ FileStatus fileStatus = fs.getFileStatus(p);
+ assertThat(fileStatus.getLen(), is((long) newLength));
+
+ AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+ fs.delete(p, false);
+ }
+
+ /**
+ * Check truncate recovery.
+ */
+ @Test
+ public void testTruncateLastBlock() throws IOException {
+ FSNamesystem fsn = cluster.getNamesystem();
+
+ String src = "/file";
+ Path srcPath = new Path(src);
+
+ byte[] contents = AppendTestUtil.initBuffer(BLOCK_SIZE);
+ writeContents(contents, BLOCK_SIZE, srcPath);
+
+ INodeFile inode = fsn.getFSDirectory().getINode(src).asFile();
+ long oldGenstamp = GenerationStamp.LAST_RESERVED_STAMP;
+ DatanodeDescriptor dn = DFSTestUtil.getLocalDatanodeDescriptor();
+ DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
+ dn.getDatanodeUuid(), InetAddress.getLocalHost().getHostAddress());
+ dn.isAlive = true;
+
+ BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
+ new Block(0, 1, oldGenstamp), (short) 1,
+ HdfsServerConstants.BlockUCState.BEING_TRUNCATED,
+ new DatanodeStorageInfo[] {storage});
+
+ inode.setBlocks(new BlockInfo[] {blockInfo});
+ fsn.writeLock();
+ try {
+ fsn.initializeBlockRecovery(inode);
+ assertThat(inode.getLastBlock().getBlockUCState(),
+ is(HdfsServerConstants.BlockUCState.BEING_TRUNCATED));
+ long blockRecoveryId = ((BlockInfoUnderConstruction) inode.getLastBlock())
+ .getBlockRecoveryId();
+ assertThat(blockRecoveryId, is(oldGenstamp + 2));
+ } finally {
+ fsn.writeUnlock();
+ }
+ }
+
+ static void writeContents(byte[] contents, int fileLength, Path p)
+ throws IOException {
+ FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
+ BLOCK_SIZE);
+ out.write(contents, 0, fileLength);
+ out.close();
+ }
+
+ static void checkBlockRecovery(Path p) throws IOException {
+ boolean success = false;
+ for(int i = 0; i < SUCCESS_ATTEMPTS; i++) {
+ LocatedBlocks blocks = getLocatedBlocks(p);
+ boolean noLastBlock = blocks.getLastLocatedBlock() == null;
+ if(!blocks.isUnderConstruction() &&
+ (noLastBlock || blocks.isLastBlockComplete())) {
+ success = true;
+ break;
+ }
+ try { Thread.sleep(SLEEP); } catch (InterruptedException ignored) {}
+ }
+ assertThat("inode should complete in ~" + SLEEP * SUCCESS_ATTEMPTS + " ms.",
+ success, is(true));
+ }
+
+ static LocatedBlocks getLocatedBlocks(Path src) throws IOException {
+ return fs.getClient().getLocatedBlocks(src.toString(), 0, Long.MAX_VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
index b9e62e3..3084f26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
@@ -409,7 +409,7 @@ public class TestNamenodeRetryCache {
LightWeightCache<CacheEntry, CacheEntry> cacheSet =
(LightWeightCache<CacheEntry, CacheEntry>) namesystem.getRetryCache().getCacheSet();
- assertEquals(23, cacheSet.size());
+ assertEquals(24, cacheSet.size());
Map<CacheEntry, CacheEntry> oldEntries =
new HashMap<CacheEntry, CacheEntry>();
@@ -428,7 +428,7 @@ public class TestNamenodeRetryCache {
assertTrue(namesystem.hasRetryCache());
cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) namesystem
.getRetryCache().getCacheSet();
- assertEquals(23, cacheSet.size());
+ assertEquals(24, cacheSet.size());
iter = cacheSet.iterator();
while (iter.hasNext()) {
CacheEntry entry = iter.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index 3739bd9..066fd66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -163,7 +163,7 @@ public class TestRetryCacheWithHA {
FSNamesystem fsn0 = cluster.getNamesystem(0);
LightWeightCache<CacheEntry, CacheEntry> cacheSet =
(LightWeightCache<CacheEntry, CacheEntry>) fsn0.getRetryCache().getCacheSet();
- assertEquals(23, cacheSet.size());
+ assertEquals(24, cacheSet.size());
Map<CacheEntry, CacheEntry> oldEntries =
new HashMap<CacheEntry, CacheEntry>();
@@ -184,7 +184,7 @@ public class TestRetryCacheWithHA {
FSNamesystem fsn1 = cluster.getNamesystem(1);
cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) fsn1
.getRetryCache().getCacheSet();
- assertEquals(23, cacheSet.size());
+ assertEquals(24, cacheSet.size());
iter = cacheSet.iterator();
while (iter.hasNext()) {
CacheEntry entry = iter.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index 08607eb..002c506 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
[13/14] hadoop git commit: HDFS-3107. Introduce truncate. Contributed
by Plamen Jeliazkov.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215c2af6/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 5e1d718..f61c075 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1412805665311</EXPIRY_DATE>
- <KEY>c1cad1109e33ae77</KEY>
+ <EXPIRY_DATE>1421822547136</EXPIRY_DATE>
+ <KEY>24319c7d1f7c0828</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1412805665314</EXPIRY_DATE>
- <KEY>0632068587d6574c</KEY>
+ <EXPIRY_DATE>1421822547140</EXPIRY_DATE>
+ <KEY>254b1207021431f4</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,19 +37,19 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114467969</MTIME>
- <ATIME>1412114467969</ATIME>
+ <MTIME>1421131348286</MTIME>
+ <ATIME>1421131348286</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>13</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>6</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -60,14 +60,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468019</MTIME>
- <ATIME>1412114467969</ATIME>
+ <MTIME>1421131348328</MTIME>
+ <ATIME>1421131348286</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -78,7 +78,7 @@
<DATA>
<TXID>6</TXID>
<PATH>/file_create</PATH>
- <POLICYID>12</POLICYID>
+ <POLICYID>7</POLICYID>
</DATA>
</RECORD>
<RECORD>
@@ -88,9 +88,9 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1412114468027</TIMESTAMP>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>16</RPC_CALLID>
+ <TIMESTAMP>1421131348343</TIMESTAMP>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>9</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -99,9 +99,9 @@
<TXID>8</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1412114468034</TIMESTAMP>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>17</RPC_CALLID>
+ <TIMESTAMP>1421131348353</TIMESTAMP>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>10</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -111,9 +111,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1412114468041</TIMESTAMP>
+ <TIMESTAMP>1421131348366</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -146,8 +146,8 @@
<TXID>13</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>22</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>15</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -157,8 +157,8 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>23</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>16</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -167,8 +167,8 @@
<TXID>15</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>24</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -179,19 +179,19 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468073</MTIME>
- <ATIME>1412114468073</ATIME>
+ <MTIME>1421131348401</MTIME>
+ <ATIME>1421131348401</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>25</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>18</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -202,14 +202,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468075</MTIME>
- <ATIME>1412114468073</ATIME>
+ <MTIME>1421131348405</MTIME>
+ <ATIME>1421131348401</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -265,10 +265,10 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1412114468093</TIMESTAMP>
+ <TIMESTAMP>1421131348436</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>32</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>25</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -279,19 +279,19 @@
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468097</MTIME>
- <ATIME>1412114468097</ATIME>
+ <MTIME>1421131348443</MTIME>
+ <ATIME>1421131348443</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>34</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>27</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -396,8 +396,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468349</MTIME>
- <ATIME>1412114468097</ATIME>
+ <MTIME>1421131348998</MTIME>
+ <ATIME>1421131348443</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -418,7 +418,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -432,19 +432,19 @@
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468351</MTIME>
- <ATIME>1412114468351</ATIME>
+ <MTIME>1421131349001</MTIME>
+ <ATIME>1421131349001</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>47</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>38</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -549,8 +549,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468370</MTIME>
- <ATIME>1412114468351</ATIME>
+ <MTIME>1421131349032</MTIME>
+ <ATIME>1421131349001</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -571,7 +571,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -585,19 +585,19 @@
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468373</MTIME>
- <ATIME>1412114468373</ATIME>
+ <MTIME>1421131349036</MTIME>
+ <ATIME>1421131349036</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>59</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>47</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -702,8 +702,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468392</MTIME>
- <ATIME>1412114468373</ATIME>
+ <MTIME>1421131349060</MTIME>
+ <ATIME>1421131349036</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -724,7 +724,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -736,80 +736,206 @@
<TXID>57</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1412114468395</TIMESTAMP>
+ <TIMESTAMP>1421131349064</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>70</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>55</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
- <OPCODE>OP_SYMLINK</OPCODE>
+ <OPCODE>OP_ADD</OPCODE>
<DATA>
<TXID>58</TXID>
<LENGTH>0</LENGTH>
<INODEID>16392</INODEID>
+ <PATH>/file_create</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1421131349068</MTIME>
+ <ATIME>1421131349068</ATIME>
+ <BLOCKSIZE>512</BLOCKSIZE>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
+ <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>true</OVERWRITE>
+ <PERMISSION_STATUS>
+ <USERNAME>plamenjeliazkov</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>57</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+ <DATA>
+ <TXID>59</TXID>
+ <BLOCK_ID>1073741834</BLOCK_ID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+ <DATA>
+ <TXID>60</TXID>
+ <GENSTAMPV2>1010</GENSTAMPV2>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ADD_BLOCK</OPCODE>
+ <DATA>
+ <TXID>61</TXID>
+ <PATH>/file_create</PATH>
+ <BLOCK>
+ <BLOCK_ID>1073741834</BLOCK_ID>
+ <NUM_BYTES>0</NUM_BYTES>
+ <GENSTAMP>1010</GENSTAMP>
+ </BLOCK>
+ <RPC_CLIENTID></RPC_CLIENTID>
+ <RPC_CALLID>-2</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+ <DATA>
+ <TXID>62</TXID>
+ <BLOCK_ID>1073741835</BLOCK_ID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+ <DATA>
+ <TXID>63</TXID>
+ <GENSTAMPV2>1011</GENSTAMPV2>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ADD_BLOCK</OPCODE>
+ <DATA>
+ <TXID>64</TXID>
+ <PATH>/file_create</PATH>
+ <BLOCK>
+ <BLOCK_ID>1073741834</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1010</GENSTAMP>
+ </BLOCK>
+ <BLOCK>
+ <BLOCK_ID>1073741835</BLOCK_ID>
+ <NUM_BYTES>0</NUM_BYTES>
+ <GENSTAMP>1011</GENSTAMP>
+ </BLOCK>
+ <RPC_CLIENTID></RPC_CLIENTID>
+ <RPC_CALLID>-2</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_CLOSE</OPCODE>
+ <DATA>
+ <TXID>65</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>0</INODEID>
+ <PATH>/file_create</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1421131349085</MTIME>
+ <ATIME>1421131349068</ATIME>
+ <BLOCKSIZE>512</BLOCKSIZE>
+ <CLIENT_NAME></CLIENT_NAME>
+ <CLIENT_MACHINE></CLIENT_MACHINE>
+ <OVERWRITE>false</OVERWRITE>
+ <BLOCK>
+ <BLOCK_ID>1073741834</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1010</GENSTAMP>
+ </BLOCK>
+ <BLOCK>
+ <BLOCK_ID>1073741835</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1011</GENSTAMP>
+ </BLOCK>
+ <PERMISSION_STATUS>
+ <USERNAME>plamenjeliazkov</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_TRUNCATE</OPCODE>
+ <DATA>
+ <TXID>66</TXID>
+ <SRC>/file_create</SRC>
+ <CLIENTNAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENTNAME>
+ <CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
+ <NEWLENGTH>512</NEWLENGTH>
+ <TIMESTAMP>1421131349088</TIMESTAMP>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SYMLINK</OPCODE>
+ <DATA>
+ <TXID>67</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>16393</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1412114468398</MTIME>
- <ATIME>1412114468398</ATIME>
+ <MTIME>1421131349095</MTIME>
+ <ATIME>1421131349095</ATIME>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>71</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>64</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>59</TXID>
+ <TXID>68</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16393</INODEID>
+ <INODEID>16394</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114468401</MTIME>
- <ATIME>1412114468401</ATIME>
+ <MTIME>1421131349098</MTIME>
+ <ATIME>1421131349098</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>72</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>65</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>60</TXID>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <TXID>69</TXID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>61</TXID>
- <GENSTAMPV2>1010</GENSTAMPV2>
+ <TXID>70</TXID>
+ <GENSTAMPV2>1012</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>62</TXID>
+ <TXID>71</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1010</GENSTAMP>
+ <GENSTAMP>1012</GENSTAMP>
</BLOCK>
<RPC_CLIENTID></RPC_CLIENTID>
<RPC_CALLID>-2</RPC_CALLID>
@@ -818,12 +944,12 @@
<RECORD>
<OPCODE>OP_UPDATE_BLOCKS</OPCODE>
<DATA>
- <TXID>63</TXID>
+ <TXID>72</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
<NUM_BYTES>11</NUM_BYTES>
- <GENSTAMP>1010</GENSTAMP>
+ <GENSTAMP>1012</GENSTAMP>
</BLOCK>
<RPC_CLIENTID></RPC_CLIENTID>
<RPC_CALLID>-2</RPC_CALLID>
@@ -832,15 +958,15 @@
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>64</TXID>
- <GENSTAMPV2>1011</GENSTAMPV2>
+ <TXID>73</TXID>
+ <GENSTAMPV2>1013</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
- <TXID>65</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_1474796918_1</LEASEHOLDER>
+ <TXID>74</TXID>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_526346936_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@@ -848,24 +974,24 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>66</TXID>
+ <TXID>75</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1412114470807</MTIME>
- <ATIME>1412114468401</ATIME>
+ <MTIME>1421131351230</MTIME>
+ <ATIME>1421131349098</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
<NUM_BYTES>11</NUM_BYTES>
- <GENSTAMP>1011</GENSTAMP>
+ <GENSTAMP>1013</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>aagarwal</USERNAME>
+ <USERNAME>plamenjeliazkov</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -874,72 +1000,72 @@
<RECORD>
<OPCODE>OP_ADD_CACHE_POOL</OPCODE>
<DATA>
- <TXID>67</TXID>
+ <TXID>76</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>aagarwal</OWNERNAME>
+ <OWNERNAME>plamenjeliazkov</OWNERNAME>
<GROUPNAME>staff</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>79</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>72</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
<DATA>
- <TXID>68</TXID>
+ <TXID>77</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>80</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>73</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>69</TXID>
+ <TXID>78</TXID>
<ID>1</ID>
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844421328165416</EXPIRATION>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>81</RPC_CALLID>
+ <EXPIRATION>2305844430345046085</EXPIRATION>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>74</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>70</TXID>
+ <TXID>79</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>82</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>75</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>71</TXID>
+ <TXID>80</TXID>
<ID>1</ID>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>83</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>76</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
<DATA>
- <TXID>72</TXID>
+ <TXID>81</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>84</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>77</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_ACL</OPCODE>
<DATA>
- <TXID>73</TXID>
+ <TXID>82</TXID>
<SRC>/file_concat_target</SRC>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
@@ -972,62 +1098,62 @@
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>74</TXID>
+ <TXID>83</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>86</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>79</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>75</TXID>
+ <TXID>84</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>87</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
- <TXID>76</TXID>
+ <TXID>85</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
</XATTR>
- <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
- <RPC_CALLID>88</RPC_CALLID>
+ <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
- <TXID>77</TXID>
- <STARTTIME>1412114471510</STARTTIME>
+ <TXID>86</TXID>
+ <STARTTIME>1421131352186</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
- <TXID>78</TXID>
- <FINALIZETIME>1412114471510</FINALIZETIME>
+ <TXID>87</TXID>
+ <FINALIZETIME>1421131352186</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
- <TXID>79</TXID>
+ <TXID>88</TXID>
</DATA>
</RECORD>
</EDITS>
[09/14] hadoop git commit: HDFS-7600. Refine hdfs admin classes to
reuse common code. Contributed by Jing Zhao.
Posted by zh...@apache.org.
HDFS-7600. Refine hdfs admin classes to reuse common code. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/210f5c55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/210f5c55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/210f5c55
Branch: refs/heads/HDFS-EC
Commit: 210f5c559a4ab51e7a02e2628f7b7585c8a029ff
Parents: 26f3b6e
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Jan 12 17:32:33 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../apache/hadoop/hdfs/tools/AdminHelper.java | 192 ++++++++++++++
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 264 +++++--------------
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 130 +--------
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 148 ++---------
5 files changed, 288 insertions(+), 448 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/210f5c55/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 487b209..cefdc16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -496,6 +496,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7598. Remove dependency on old version of guava in
TestDFSClientCache#testEviction. (Sangjin Lee via Colin P. McCabe)
+ HDFS-7600. Refine hdfs admin classes to reuse common code. (jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/210f5c55/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
new file mode 100644
index 0000000..153fb36
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.tools.TableListing;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Helper methods for CacheAdmin/CryptoAdmin/StoragePolicyAdmin
+ */
+public class AdminHelper {
+ /**
+ * Maximum length for printed lines
+ */
+ static final int MAX_LINE_WIDTH = 80;
+ static final String HELP_COMMAND_NAME = "-help";
+
+ static DistributedFileSystem getDFS(Configuration conf)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ if (!(fs instanceof DistributedFileSystem)) {
+ throw new IllegalArgumentException("FileSystem " + fs.getUri() +
+ " is not an HDFS file system");
+ }
+ return (DistributedFileSystem)fs;
+ }
+
+ /**
+ * NN exceptions contain the stack trace as part of the exception message.
+ * When it's a known error, pretty-print the error and squish the stack trace.
+ */
+ static String prettifyException(Exception e) {
+ return e.getClass().getSimpleName() + ": "
+ + e.getLocalizedMessage().split("\n")[0];
+ }
+
+ static TableListing getOptionDescriptionListing() {
+ return new TableListing.Builder()
+ .addField("").addField("", true)
+ .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
+ }
+
+ /**
+ * Parses a time-to-live value from a string
+ * @return The ttl in milliseconds
+ * @throws IOException if it could not be parsed
+ */
+ static Long parseTtlString(String maxTtlString) throws IOException {
+ Long maxTtl = null;
+ if (maxTtlString != null) {
+ if (maxTtlString.equalsIgnoreCase("never")) {
+ maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER;
+ } else {
+ maxTtl = DFSUtil.parseRelativeTime(maxTtlString);
+ }
+ }
+ return maxTtl;
+ }
+
+ static Long parseLimitString(String limitString) {
+ Long limit = null;
+ if (limitString != null) {
+ if (limitString.equalsIgnoreCase("unlimited")) {
+ limit = CachePoolInfo.LIMIT_UNLIMITED;
+ } else {
+ limit = Long.parseLong(limitString);
+ }
+ }
+ return limit;
+ }
+
+ static Command determineCommand(String commandName, Command[] commands) {
+ Preconditions.checkNotNull(commands);
+ if (HELP_COMMAND_NAME.equals(commandName)) {
+ return new HelpCommand(commands);
+ }
+ for (Command command : commands) {
+ if (command.getName().equals(commandName)) {
+ return command;
+ }
+ }
+ return null;
+ }
+
+ static void printUsage(boolean longUsage, String toolName,
+ Command[] commands) {
+ Preconditions.checkNotNull(commands);
+ System.err.println("Usage: bin/hdfs " + toolName + " [COMMAND]");
+ final HelpCommand helpCommand = new HelpCommand(commands);
+ for (AdminHelper.Command command : commands) {
+ if (longUsage) {
+ System.err.print(command.getLongUsage());
+ } else {
+ System.err.print(" " + command.getShortUsage());
+ }
+ }
+ System.err.print(longUsage ? helpCommand.getLongUsage() :
+ (" " + helpCommand.getShortUsage()));
+ System.err.println();
+ }
+
+ interface Command {
+ String getName();
+ String getShortUsage();
+ String getLongUsage();
+ int run(Configuration conf, List<String> args) throws IOException;
+ }
+
+ static class HelpCommand implements Command {
+ private final Command[] commands;
+
+ public HelpCommand(Command[] commands) {
+ Preconditions.checkNotNull(commands != null);
+ this.commands = commands;
+ }
+
+ @Override
+ public String getName() {
+ return HELP_COMMAND_NAME;
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[-help <command-name>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<command-name>", "The command for which to get " +
+ "detailed help. If no command is specified, print detailed help for " +
+ "all commands");
+ return getShortUsage() + "\n" +
+ "Get detailed help about a command.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ if (args.size() == 0) {
+ for (AdminHelper.Command command : commands) {
+ System.err.println(command.getLongUsage());
+ }
+ return 0;
+ }
+ if (args.size() != 1) {
+ System.out.println("You must give exactly one argument to -help.");
+ return 0;
+ }
+ final String commandName = args.get(0);
+ // prepend a dash to match against the command names
+ final AdminHelper.Command command = AdminHelper
+ .determineCommand("-" + commandName, commands);
+ if (command == null) {
+ System.err.print("Unknown command '" + commandName + "'.\n");
+ System.err.print("Valid help command names are:\n");
+ String separator = "";
+ for (AdminHelper.Command c : commands) {
+ System.err.print(separator + c.getName().substring(1));
+ separator = ", ";
+ }
+ System.err.print("\n");
+ return 1;
+ }
+ System.err.print(command.getLongUsage());
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/210f5c55/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index dbe2284..6888ea8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CacheFlag;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -53,11 +52,6 @@ import com.google.common.base.Joiner;
@InterfaceAudience.Private
public class CacheAdmin extends Configured implements Tool {
- /**
- * Maximum length for printed lines
- */
- private static final int MAX_LINE_WIDTH = 80;
-
public CacheAdmin() {
this(null);
}
@@ -69,16 +63,17 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public int run(String[] args) throws IOException {
if (args.length == 0) {
- printUsage(false);
+ AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
return 1;
}
- Command command = determineCommand(args[0]);
+ AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+ COMMANDS);
if (command == null) {
System.err.println("Can't understand command '" + args[0] + "'");
if (!args[0].startsWith("-")) {
System.err.println("Command names must start with dashes.");
}
- printUsage(false);
+ AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
return 1;
}
List<String> argsList = new LinkedList<String>();
@@ -88,7 +83,7 @@ public class CacheAdmin extends Configured implements Tool {
try {
return command.run(getConf(), argsList);
} catch (IllegalArgumentException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return -1;
}
}
@@ -98,64 +93,9 @@ public class CacheAdmin extends Configured implements Tool {
System.exit(cacheAdmin.run(argsArray));
}
- private static DistributedFileSystem getDFS(Configuration conf)
+ private static CacheDirectiveInfo.Expiration parseExpirationString(String ttlString)
throws IOException {
- FileSystem fs = FileSystem.get(conf);
- if (!(fs instanceof DistributedFileSystem)) {
- throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not an HDFS file system");
- }
- return (DistributedFileSystem)fs;
- }
-
- /**
- * NN exceptions contain the stack trace as part of the exception message.
- * When it's a known error, pretty-print the error and squish the stack trace.
- */
- private static String prettifyException(Exception e) {
- return e.getClass().getSimpleName() + ": "
- + e.getLocalizedMessage().split("\n")[0];
- }
-
- private static TableListing getOptionDescriptionListing() {
- TableListing listing = new TableListing.Builder()
- .addField("").addField("", true)
- .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
- return listing;
- }
-
- /**
- * Parses a time-to-live value from a string
- * @return The ttl in milliseconds
- * @throws IOException if it could not be parsed
- */
- private static Long parseTtlString(String maxTtlString) throws IOException {
- Long maxTtl = null;
- if (maxTtlString != null) {
- if (maxTtlString.equalsIgnoreCase("never")) {
- maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER;
- } else {
- maxTtl = DFSUtil.parseRelativeTime(maxTtlString);
- }
- }
- return maxTtl;
- }
-
- private static Long parseLimitString(String limitString) {
- Long limit = null;
- if (limitString != null) {
- if (limitString.equalsIgnoreCase("unlimited")) {
- limit = CachePoolInfo.LIMIT_UNLIMITED;
- } else {
- limit = Long.parseLong(limitString);
- }
- }
- return limit;
- }
-
- private static Expiration parseExpirationString(String ttlString)
- throws IOException {
- Expiration ex = null;
+ CacheDirectiveInfo.Expiration ex = null;
if (ttlString != null) {
if (ttlString.equalsIgnoreCase("never")) {
ex = CacheDirectiveInfo.Expiration.NEVER;
@@ -167,14 +107,8 @@ public class CacheAdmin extends Configured implements Tool {
return ex;
}
- interface Command {
- String getName();
- String getShortUsage();
- String getLongUsage();
- int run(Configuration conf, List<String> args) throws IOException;
- }
-
- private static class AddCacheDirectiveInfoCommand implements Command {
+ private static class AddCacheDirectiveInfoCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-addDirective";
@@ -190,7 +124,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<path>", "A path to cache. The path can be " +
"a directory or a file.");
listing.addRow("<pool-name>", "The pool to which the directive will be " +
@@ -252,7 +186,7 @@ public class CacheAdmin extends Configured implements Tool {
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
CacheDirectiveInfo directive = builder.build();
EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
if (force) {
@@ -262,7 +196,7 @@ public class CacheAdmin extends Configured implements Tool {
long id = dfs.addCacheDirective(directive, flags);
System.out.println("Added cache directive " + id);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
@@ -270,7 +204,8 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class RemoveCacheDirectiveInfoCommand implements Command {
+ private static class RemoveCacheDirectiveInfoCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-removeDirective";
@@ -283,7 +218,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<id>", "The id of the cache directive to remove. " +
"You must have write permission on the pool of the " +
"directive in order to remove it. To see a list " +
@@ -318,19 +253,20 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Usage is " + getShortUsage());
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.getClient().removeCacheDirective(id);
System.out.println("Removed cached directive " + id);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
}
}
- private static class ModifyCacheDirectiveInfoCommand implements Command {
+ private static class ModifyCacheDirectiveInfoCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-modifyDirective";
@@ -345,7 +281,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<id>", "The ID of the directive to modify (required)");
listing.addRow("<path>", "A path to cache. The path can be " +
"a directory or a file. (optional)");
@@ -415,7 +351,7 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("No modifications were specified.");
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
if (force) {
flags.add(CacheFlag.FORCE);
@@ -424,14 +360,15 @@ public class CacheAdmin extends Configured implements Tool {
dfs.modifyCacheDirective(builder.build(), flags);
System.out.println("Modified cache directive " + idString);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
}
}
- private static class RemoveCacheDirectiveInfosCommand implements Command {
+ private static class RemoveCacheDirectiveInfosCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-removeDirectives";
@@ -444,7 +381,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("-path <path>", "The path of the cache directives to remove. " +
"You must have write permission on the pool of the directive in order " +
"to remove it. To see a list of cache directives, use the " +
@@ -468,7 +405,7 @@ public class CacheAdmin extends Configured implements Tool {
}
int exitCode = 0;
try {
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
RemoteIterator<CacheDirectiveEntry> iter =
dfs.listCacheDirectives(
new CacheDirectiveInfo.Builder().
@@ -480,12 +417,12 @@ public class CacheAdmin extends Configured implements Tool {
System.out.println("Removed cache directive " +
entry.getInfo().getId());
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
exitCode = 2;
}
}
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
exitCode = 2;
}
if (exitCode == 0) {
@@ -496,7 +433,8 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class ListCacheDirectiveInfoCommand implements Command {
+ private static class ListCacheDirectiveInfoCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-listDirectives";
@@ -510,7 +448,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("-stats", "List path-based cache directive statistics.");
listing.addRow("<path>", "List only " +
"cache directives with this path. " +
@@ -559,7 +497,7 @@ public class CacheAdmin extends Configured implements Tool {
}
TableListing tableListing = tableBuilder.build();
try {
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
RemoteIterator<CacheDirectiveEntry> iter =
dfs.listCacheDirectives(builder.build());
int numEntries = 0;
@@ -587,7 +525,7 @@ public class CacheAdmin extends Configured implements Tool {
row.add("" + stats.getFilesNeeded());
row.add("" + stats.getFilesCached());
}
- tableListing.addRow(row.toArray(new String[0]));
+ tableListing.addRow(row.toArray(new String[row.size()]));
numEntries++;
}
System.out.print(String.format("Found %d entr%s%n",
@@ -596,14 +534,14 @@ public class CacheAdmin extends Configured implements Tool {
System.out.print(tableListing);
}
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
}
}
- private static class AddCachePoolCommand implements Command {
+ private static class AddCachePoolCommand implements AdminHelper.Command {
private static final String NAME = "-addPool";
@@ -621,7 +559,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<name>", "Name of the new pool.");
listing.addRow("<owner>", "Username of the owner of the pool. " +
@@ -669,13 +607,13 @@ public class CacheAdmin extends Configured implements Tool {
info.setMode(new FsPermission(mode));
}
String limitString = StringUtils.popOptionWithArgument("-limit", args);
- Long limit = parseLimitString(limitString);
+ Long limit = AdminHelper.parseLimitString(limitString);
if (limit != null) {
info.setLimit(limit);
}
String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
try {
- Long maxTtl = parseTtlString(maxTtlString);
+ Long maxTtl = AdminHelper.parseTtlString(maxTtlString);
if (maxTtl != null) {
info.setMaxRelativeExpiryMs(maxTtl);
}
@@ -691,11 +629,11 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Usage is " + getShortUsage());
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.addCachePool(info);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
System.out.println("Successfully added cache pool " + name + ".");
@@ -703,7 +641,7 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class ModifyCachePoolCommand implements Command {
+ private static class ModifyCachePoolCommand implements AdminHelper.Command {
@Override
public String getName() {
@@ -719,7 +657,7 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<name>", "Name of the pool to modify.");
listing.addRow("<owner>", "Username of the owner of the pool");
@@ -733,7 +671,7 @@ public class CacheAdmin extends Configured implements Tool {
return getShortUsage() + "\n" +
WordUtils.wrap("Modifies the metadata of an existing cache pool. " +
"See usage of " + AddCachePoolCommand.NAME + " for more details.",
- MAX_LINE_WIDTH) + "\n\n" +
+ AdminHelper.MAX_LINE_WIDTH) + "\n\n" +
listing.toString();
}
@@ -745,11 +683,11 @@ public class CacheAdmin extends Configured implements Tool {
Integer mode = (modeString == null) ?
null : Integer.parseInt(modeString, 8);
String limitString = StringUtils.popOptionWithArgument("-limit", args);
- Long limit = parseLimitString(limitString);
+ Long limit = AdminHelper.parseLimitString(limitString);
String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
- Long maxTtl = null;
+ Long maxTtl;
try {
- maxTtl = parseTtlString(maxTtlString);
+ maxTtl = AdminHelper.parseTtlString(maxTtlString);
} catch (IOException e) {
System.err.println(
"Error while parsing maxTtl value: " + e.getMessage());
@@ -794,11 +732,11 @@ public class CacheAdmin extends Configured implements Tool {
"change in the cache pool.");
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.modifyCachePool(info);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
System.out.print("Successfully modified cache pool " + name);
@@ -827,7 +765,7 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class RemoveCachePoolCommand implements Command {
+ private static class RemoveCachePoolCommand implements AdminHelper.Command {
@Override
public String getName() {
@@ -843,7 +781,7 @@ public class CacheAdmin extends Configured implements Tool {
public String getLongUsage() {
return getShortUsage() + "\n" +
WordUtils.wrap("Remove a cache pool. This also uncaches paths " +
- "associated with the pool.\n\n", MAX_LINE_WIDTH) +
+ "associated with the pool.\n\n", AdminHelper.MAX_LINE_WIDTH) +
"<name> Name of the cache pool to remove.\n";
}
@@ -861,11 +799,11 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Usage is " + getShortUsage());
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.removeCachePool(name);
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
System.out.println("Successfully removed cache pool " + name + ".");
@@ -873,7 +811,7 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class ListCachePoolsCommand implements Command {
+ private static class ListCachePoolsCommand implements AdminHelper.Command {
@Override
public String getName() {
@@ -887,15 +825,14 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("-stats", "Display additional cache pool statistics.");
listing.addRow("<name>", "If specified, list only the named cache pool.");
return getShortUsage() + "\n" +
WordUtils.wrap("Display information about one or more cache pools, " +
- "e.g. name, owner, group, permissions, etc.", MAX_LINE_WIDTH) +
- "\n\n" +
- listing.toString();
+ "e.g. name, owner, group, permissions, etc.",
+ AdminHelper.MAX_LINE_WIDTH) + "\n\n" + listing.toString();
}
@Override
@@ -908,7 +845,7 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Usage is " + getShortUsage());
return 1;
}
- DistributedFileSystem dfs = getDFS(conf);
+ DistributedFileSystem dfs = AdminHelper.getDFS(conf);
TableListing.Builder builder = new TableListing.Builder().
addField("NAME", Justification.LEFT).
addField("OWNER", Justification.LEFT).
@@ -949,7 +886,7 @@ public class CacheAdmin extends Configured implements Tool {
String maxTtlString = null;
if (maxTtl != null) {
- if (maxTtl.longValue() == CachePoolInfo.RELATIVE_EXPIRY_NEVER) {
+ if (maxTtl == CachePoolInfo.RELATIVE_EXPIRY_NEVER) {
maxTtlString = "never";
} else {
maxTtlString = DFSUtil.durationToString(maxTtl);
@@ -964,7 +901,7 @@ public class CacheAdmin extends Configured implements Tool {
row.add(Long.toString(stats.getFilesNeeded()));
row.add(Long.toString(stats.getFilesCached()));
}
- listing.addRow(row.toArray(new String[] {}));
+ listing.addRow(row.toArray(new String[row.size()]));
++numResults;
if (name != null) {
break;
@@ -972,7 +909,7 @@ public class CacheAdmin extends Configured implements Tool {
}
}
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
System.out.print(String.format("Found %d result%s.%n", numResults,
@@ -985,61 +922,7 @@ public class CacheAdmin extends Configured implements Tool {
}
}
- private static class HelpCommand implements Command {
- @Override
- public String getName() {
- return "-help";
- }
-
- @Override
- public String getShortUsage() {
- return "[-help <command-name>]\n";
- }
-
- @Override
- public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
- listing.addRow("<command-name>", "The command for which to get " +
- "detailed help. If no command is specified, print detailed help for " +
- "all commands");
- return getShortUsage() + "\n" +
- "Get detailed help about a command.\n\n" +
- listing.toString();
- }
-
- @Override
- public int run(Configuration conf, List<String> args) throws IOException {
- if (args.size() == 0) {
- for (Command command : COMMANDS) {
- System.err.println(command.getLongUsage());
- }
- return 0;
- }
- if (args.size() != 1) {
- System.out.println("You must give exactly one argument to -help.");
- return 0;
- }
- String commandName = args.get(0);
- // prepend a dash to match against the command names
- Command command = determineCommand("-"+commandName);
- if (command == null) {
- System.err.print("Sorry, I don't know the command '" +
- commandName + "'.\n");
- System.err.print("Valid help command names are:\n");
- String separator = "";
- for (Command c : COMMANDS) {
- System.err.print(separator + c.getName().substring(1));
- separator = ", ";
- }
- System.err.print("\n");
- return 1;
- }
- System.err.print(command.getLongUsage());
- return 0;
- }
- }
-
- private static final Command[] COMMANDS = {
+ private static final AdminHelper.Command[] COMMANDS = {
new AddCacheDirectiveInfoCommand(),
new ModifyCacheDirectiveInfoCommand(),
new ListCacheDirectiveInfoCommand(),
@@ -1048,29 +931,6 @@ public class CacheAdmin extends Configured implements Tool {
new AddCachePoolCommand(),
new ModifyCachePoolCommand(),
new RemoveCachePoolCommand(),
- new ListCachePoolsCommand(),
- new HelpCommand(),
+ new ListCachePoolsCommand()
};
-
- private static void printUsage(boolean longUsage) {
- System.err.println(
- "Usage: bin/hdfs cacheadmin [COMMAND]");
- for (Command command : COMMANDS) {
- if (longUsage) {
- System.err.print(command.getLongUsage());
- } else {
- System.err.print(" " + command.getShortUsage());
- }
- }
- System.err.println();
- }
-
- private static Command determineCommand(String commandName) {
- for (int i = 0; i < COMMANDS.length; i++) {
- if (COMMANDS[i].getName().equals(commandName)) {
- return COMMANDS[i];
- }
- }
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/210f5c55/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index bb52ddd..09825b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -39,11 +38,6 @@ import org.apache.hadoop.util.Tool;
@InterfaceAudience.Private
public class CryptoAdmin extends Configured implements Tool {
- /**
- * Maximum length for printed lines
- */
- private static final int MAX_LINE_WIDTH = 80;
-
public CryptoAdmin() {
this(null);
}
@@ -55,16 +49,17 @@ public class CryptoAdmin extends Configured implements Tool {
@Override
public int run(String[] args) throws IOException {
if (args.length == 0) {
- printUsage(false);
+ AdminHelper.printUsage(false, "crypto", COMMANDS);
return 1;
}
- final Command command = determineCommand(args[0]);
+ final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+ COMMANDS);
if (command == null) {
System.err.println("Can't understand command '" + args[0] + "'");
if (!args[0].startsWith("-")) {
System.err.println("Command names must start with dashes.");
}
- printUsage(false);
+ AdminHelper.printUsage(false, "crypto", COMMANDS);
return 1;
}
final List<String> argsList = new LinkedList<String>();
@@ -84,16 +79,6 @@ public class CryptoAdmin extends Configured implements Tool {
System.exit(cryptoAdmin.run(argsArray));
}
- private static DistributedFileSystem getDFS(Configuration conf)
- throws IOException {
- final FileSystem fs = FileSystem.get(conf);
- if (!(fs instanceof DistributedFileSystem)) {
- throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not an HDFS file system");
- }
- return (DistributedFileSystem) fs;
- }
-
/**
* NN exceptions contain the stack trace as part of the exception message.
* When it's a known error, pretty-print the error and squish the stack trace.
@@ -103,21 +88,7 @@ public class CryptoAdmin extends Configured implements Tool {
e.getLocalizedMessage().split("\n")[0];
}
- private static TableListing getOptionDescriptionListing() {
- final TableListing listing = new TableListing.Builder()
- .addField("").addField("", true)
- .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
- return listing;
- }
-
- interface Command {
- String getName();
- String getShortUsage();
- String getLongUsage();
- int run(Configuration conf, List<String> args) throws IOException;
- }
-
- private static class CreateZoneCommand implements Command {
+ private static class CreateZoneCommand implements AdminHelper.Command {
@Override
public String getName() {
return "-createZone";
@@ -130,7 +101,7 @@ public class CryptoAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- final TableListing listing = getOptionDescriptionListing();
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<path>", "The path of the encryption zone to create. " +
"It must be an empty directory.");
listing.addRow("<keyName>", "Name of the key to use for the " +
@@ -160,7 +131,7 @@ public class CryptoAdmin extends Configured implements Tool {
return 1;
}
- final DistributedFileSystem dfs = getDFS(conf);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.createEncryptionZone(new Path(path), keyName);
System.out.println("Added encryption zone " + path);
@@ -173,7 +144,7 @@ public class CryptoAdmin extends Configured implements Tool {
}
}
- private static class ListZonesCommand implements Command {
+ private static class ListZonesCommand implements AdminHelper.Command {
@Override
public String getName() {
return "-listZones";
@@ -197,11 +168,11 @@ public class CryptoAdmin extends Configured implements Tool {
return 1;
}
- final DistributedFileSystem dfs = getDFS(conf);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
final TableListing listing = new TableListing.Builder()
.addField("").addField("", true)
- .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
+ .wrapWidth(AdminHelper.MAX_LINE_WIDTH).hideHeaders().build();
final RemoteIterator<EncryptionZone> it = dfs.listEncryptionZones();
while (it.hasNext()) {
EncryptionZone ez = it.next();
@@ -217,85 +188,8 @@ public class CryptoAdmin extends Configured implements Tool {
}
}
- private static class HelpCommand implements Command {
- @Override
- public String getName() {
- return "-help";
- }
-
- @Override
- public String getShortUsage() {
- return "[-help <command-name>]\n";
- }
-
- @Override
- public String getLongUsage() {
- final TableListing listing = getOptionDescriptionListing();
- listing.addRow("<command-name>", "The command for which to get " +
- "detailed help. If no command is specified, print detailed help for " +
- "all commands");
- return getShortUsage() + "\n" +
- "Get detailed help about a command.\n\n" +
- listing.toString();
- }
-
- @Override
- public int run(Configuration conf, List<String> args) throws IOException {
- if (args.size() == 0) {
- for (Command command : COMMANDS) {
- System.err.println(command.getLongUsage());
- }
- return 0;
- }
- if (args.size() != 1) {
- System.out.println("You must give exactly one argument to -help.");
- return 0;
- }
- final String commandName = args.get(0);
- // prepend a dash to match against the command names
- final Command command = determineCommand("-"+commandName);
- if (command == null) {
- System.err.print("Sorry, I don't know the command '" +
- commandName + "'.\n");
- System.err.print("Valid help command names are:\n");
- String separator = "";
- for (Command c : COMMANDS) {
- System.err.print(separator + c.getName().substring(1));
- separator = ", ";
- }
- System.err.print("\n");
- return 1;
- }
- System.err.print(command.getLongUsage());
- return 0;
- }
- }
-
- private static final Command[] COMMANDS = {
+ private static final AdminHelper.Command[] COMMANDS = {
new CreateZoneCommand(),
- new ListZonesCommand(),
- new HelpCommand(),
+ new ListZonesCommand()
};
-
- private static void printUsage(boolean longUsage) {
- System.err.println(
- "Usage: bin/hdfs crypto [COMMAND]");
- for (Command command : COMMANDS) {
- if (longUsage) {
- System.err.print(command.getLongUsage());
- } else {
- System.err.print(" " + command.getShortUsage());
- }
- }
- System.err.println();
- }
-
- private static Command determineCommand(String commandName) {
- for (int i = 0; i < COMMANDS.length; i++) {
- if (COMMANDS[i].getName().equals(commandName)) {
- return COMMANDS[i];
- }
- }
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/210f5c55/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 6023e40..d1b6017 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.tools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -38,7 +37,6 @@ import java.util.List;
* This class implements block storage policy operations.
*/
public class StoragePolicyAdmin extends Configured implements Tool {
- private static final int MAX_LINE_WIDTH = 80;
public static void main(String[] argsArray) throws Exception {
final StoragePolicyAdmin admin = new StoragePolicyAdmin(new
@@ -46,31 +44,6 @@ public class StoragePolicyAdmin extends Configured implements Tool {
System.exit(admin.run(argsArray));
}
- private static DistributedFileSystem getDFS(Configuration conf)
- throws IOException {
- final FileSystem fs = FileSystem.get(conf);
- if (!(fs instanceof DistributedFileSystem)) {
- throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not an HDFS file system");
- }
- return (DistributedFileSystem) fs;
- }
-
- /**
- * NN exceptions contain the stack trace as part of the exception message.
- * When it's a known error, pretty-print the error and squish the stack trace.
- */
- private static String prettifyException(Exception e) {
- return e.getClass().getSimpleName() + ": " +
- e.getLocalizedMessage().split("\n")[0];
- }
-
- private static TableListing getOptionDescriptionListing() {
- return new TableListing.Builder()
- .addField("").addField("", true)
- .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
- }
-
public StoragePolicyAdmin(Configuration conf) {
super(conf);
}
@@ -78,37 +51,32 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
if (args.length == 0) {
- printUsage(false);
+ AdminHelper.printUsage(false, "storagepolicies", COMMANDS);
return 1;
}
- final Command command = determineCommand(args[0]);
+ final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+ COMMANDS);
if (command == null) {
System.err.println("Can't understand command '" + args[0] + "'");
if (!args[0].startsWith("-")) {
System.err.println("Command names must start with dashes.");
}
- printUsage(false);
+ AdminHelper.printUsage(false, "storagepolicies", COMMANDS);
return 1;
}
- final List<String> argsList = new LinkedList<String>();
+ final List<String> argsList = new LinkedList<>();
argsList.addAll(Arrays.asList(args).subList(1, args.length));
try {
return command.run(getConf(), argsList);
} catch (IllegalArgumentException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return -1;
}
}
- interface Command {
- String getName();
- String getShortUsage();
- String getLongUsage();
- int run(Configuration conf, List<String> args) throws IOException;
- }
-
/** Command to list all the existing storage policies */
- private static class ListStoragePoliciesCommand implements Command {
+ private static class ListStoragePoliciesCommand
+ implements AdminHelper.Command {
@Override
public String getName() {
return "-listPolicies";
@@ -127,7 +95,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public int run(Configuration conf, List<String> args) throws IOException {
- final DistributedFileSystem dfs = getDFS(conf);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
BlockStoragePolicy[] policies = dfs.getStoragePolicies();
System.out.println("Block Storage Policies:");
@@ -137,7 +105,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
}
} catch (IOException e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
@@ -145,7 +113,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
/** Command to get the storage policy of a file/directory */
- private static class GetStoragePolicyCommand implements Command {
+ private static class GetStoragePolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
return "-getStoragePolicy";
@@ -158,7 +126,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- final TableListing listing = getOptionDescriptionListing();
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<path>",
"The path of the file/directory for getting the storage policy");
return getShortUsage() + "\n" +
@@ -175,7 +143,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
return 1;
}
- final DistributedFileSystem dfs = getDFS(conf);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
HdfsFileStatus status = dfs.getClient().getFileInfo(path);
if (status == null) {
@@ -195,7 +163,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
}
} catch (Exception e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
System.err.println("Cannot identify the storage policy for " + path);
@@ -204,7 +172,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
/** Command to set the storage policy to a file/directory */
- private static class SetStoragePolicyCommand implements Command {
+ private static class SetStoragePolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
return "-setStoragePolicy";
@@ -217,7 +185,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public String getLongUsage() {
- TableListing listing = getOptionDescriptionListing();
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<path>", "The path of the file/directory to set storage" +
" policy");
listing.addRow("<policy>", "The name of the block storage policy");
@@ -243,97 +211,21 @@ public class StoragePolicyAdmin extends Configured implements Tool {
return 1;
}
- final DistributedFileSystem dfs = getDFS(conf);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.setStoragePolicy(new Path(path), policyName);
System.out.println("Set storage policy " + policyName + " on " + path);
} catch (Exception e) {
- System.err.println(prettifyException(e));
+ System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
}
}
- private static class HelpCommand implements Command {
- @Override
- public String getName() {
- return "-help";
- }
-
- @Override
- public String getShortUsage() {
- return "[-help <command-name>]\n";
- }
-
- @Override
- public String getLongUsage() {
- final TableListing listing = getOptionDescriptionListing();
- listing.addRow("<command-name>", "The command for which to get " +
- "detailed help. If no command is specified, print detailed help for " +
- "all commands");
- return getShortUsage() + "\n" +
- "Get detailed help about a command.\n\n" +
- listing.toString();
- }
-
- @Override
- public int run(Configuration conf, List<String> args) throws IOException {
- if (args.size() == 0) {
- for (Command command : COMMANDS) {
- System.err.println(command.getLongUsage());
- }
- return 0;
- }
- if (args.size() != 1) {
- System.out.println("You must give exactly one argument to -help.");
- return 0;
- }
- final String commandName = args.get(0);
- // prepend a dash to match against the command names
- final Command command = determineCommand("-" + commandName);
- if (command == null) {
- System.err.print("Unknown command '" + commandName + "'.\n");
- System.err.print("Valid help command names are:\n");
- String separator = "";
- for (Command c : COMMANDS) {
- System.err.print(separator + c.getName().substring(1));
- separator = ", ";
- }
- System.err.print("\n");
- return 1;
- }
- System.err.print(command.getLongUsage());
- return 0;
- }
- }
-
- private static final Command[] COMMANDS = {
+ private static final AdminHelper.Command[] COMMANDS = {
new ListStoragePoliciesCommand(),
new SetStoragePolicyCommand(),
- new GetStoragePolicyCommand(),
- new HelpCommand()
+ new GetStoragePolicyCommand()
};
-
- private static void printUsage(boolean longUsage) {
- System.err.println(
- "Usage: bin/hdfs storagepolicies [COMMAND]");
- for (Command command : COMMANDS) {
- if (longUsage) {
- System.err.print(command.getLongUsage());
- } else {
- System.err.print(" " + command.getShortUsage());
- }
- }
- System.err.println();
- }
-
- private static Command determineCommand(String commandName) {
- for (Command COMMAND : COMMANDS) {
- if (COMMAND.getName().equals(commandName)) {
- return COMMAND;
- }
- }
- return null;
- }
}
\ No newline at end of file
[05/14] hadoop git commit: HDFS-5445. PacketReceiver populates the
packetLen field in PacketHeader incorrectly (Jonathan Mace via Colin P.
McCabe)
Posted by zh...@apache.org.
HDFS-5445. PacketReceiver populates the packetLen field in PacketHeader incorrectly (Jonathan Mace via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26f3b6e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26f3b6e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26f3b6e8
Branch: refs/heads/HDFS-EC
Commit: 26f3b6e88f47bedbf9cf77396bdabfbac66c7d31
Parents: db2abfc
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jan 12 17:11:03 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Jan 13 07:33:17 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/protocol/datatransfer/PacketReceiver.java | 2 +-
.../hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java | 5 ++++-
3 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26f3b6e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b57503a..487b209 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -679,6 +679,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7533. Datanode sometimes does not shutdown on receiving upgrade
shutdown command (Eric Payne via kihwal)
+ HDFS-5445. PacketReceiver populates the packetLen field in PacketHeader
+ incorrectly (Jonathan Mace via Colin P. McCabe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26f3b6e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
index 0de445c..3045a13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
@@ -178,7 +178,7 @@ public class PacketReceiver implements Closeable {
if (curHeader == null) {
curHeader = new PacketHeader();
}
- curHeader.setFieldsFromData(dataPlusChecksumLen, headerBuf);
+ curHeader.setFieldsFromData(payloadLen, headerBuf);
// Compute the sub-slices of the packet
int checksumLen = dataPlusChecksumLen - curHeader.getDataLen();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26f3b6e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
index 6a87cdf..64ca3fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.AppendTestUtil;
import org.junit.Test;
import org.mockito.Mockito;
+import com.google.common.primitives.Ints;
+
import static org.junit.Assert.*;
public class TestPacketReceiver {
@@ -38,7 +40,7 @@ public class TestPacketReceiver {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
- int packetLen = data.length + sums.length + 4;
+ int packetLen = data.length + sums.length + Ints.BYTES;
PacketHeader header = new PacketHeader(
packetLen, OFFSET_IN_BLOCK, SEQNO, false, data.length, false);
header.write(dos);
@@ -87,6 +89,7 @@ public class TestPacketReceiver {
PacketHeader header = pr.getHeader();
assertEquals(SEQNO, header.getSeqno());
assertEquals(OFFSET_IN_BLOCK, header.getOffsetInBlock());
+ assertEquals(dataLen + checksumsLen + Ints.BYTES, header.getPacketLen());
// Mirror the packet to an output stream and make sure it matches
// the packet we sent.