You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/03/04 02:01:40 UTC
[01/50] [abbrv] hadoop git commit: HDFS-9710. DN can be configured to
send block receipt IBRs in batches.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-1312 3bb9d5e8e -> 8e85174be
HDFS-9710. DN can be configured to send block receipt IBRs in batches.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1d4e166
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1d4e166
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1d4e166
Branch: refs/heads/HDFS-1312
Commit: d1d4e16690cc85f7f22fbead9cf596260819b561
Parents: eab52df
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Feb 26 15:32:25 2016 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Feb 26 15:32:25 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../hdfs/server/datanode/BPOfferService.java | 17 +-
.../hdfs/server/datanode/BPServiceActor.java | 19 +-
.../hdfs/server/datanode/BlockReceiver.java | 8 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 4 +
.../hadoop/hdfs/server/datanode/DataNode.java | 26 +-
.../hdfs/server/datanode/DataXceiver.java | 14 +-
.../datanode/IncrementalBlockReportManager.java | 51 +++-
.../server/datanode/fsdataset/FsDatasetSpi.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 +-
.../protocol/StorageReceivedDeletedBlocks.java | 7 +
.../server/datanode/SimulatedFSDataset.java | 11 +-
.../server/datanode/TestBPOfferService.java | 2 +-
.../hdfs/server/datanode/TestBatchIbr.java | 263 +++++++++++++++++++
.../TestDataXceiverLazyPersistHint.java | 6 +-
.../datanode/TestIncrementalBlockReports.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 4 +-
.../impl/TestInterDatanodeProtocol.java | 6 +-
19 files changed, 388 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0f1c45d..81b48c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2058,6 +2058,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9425. Expose number of blocks per volume as a metric
(Brahma Reddy Battula via vinayakumarb)
+ HDFS-9710. DN can be configured to send block receipt IBRs in batches.
+ (szetszwo)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 2c9db46..bfb6203 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -555,6 +555,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DF_INTERVAL_KEY = "dfs.df.interval";
public static final int DFS_DF_INTERVAL_DEFAULT = 60000;
+ public static final String DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY
+ = "dfs.blockreport.incremental.intervalMsec";
+ public static final long DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_DEFAULT
+ = 0;
public static final String DFS_BLOCKREPORT_INTERVAL_MSEC_KEY = "dfs.blockreport.intervalMsec";
public static final long DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT = 6 * 60 * 60 * 1000;
public static final String DFS_BLOCKREPORT_INITIAL_DELAY_KEY = "dfs.blockreport.initialDelay";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index cc536d3..206e89a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -233,29 +233,32 @@ class BPOfferService {
* till namenode is informed before responding with success to the
* client? For now we don't.
*/
- void notifyNamenodeReceivedBlock(
- ExtendedBlock block, String delHint, String storageUuid) {
+ void notifyNamenodeReceivedBlock(ExtendedBlock block, String delHint,
+ String storageUuid, boolean isOnTransientStorage) {
notifyNamenodeBlock(block, BlockStatus.RECEIVED_BLOCK, delHint,
- storageUuid);
+ storageUuid, isOnTransientStorage);
}
void notifyNamenodeReceivingBlock(ExtendedBlock block, String storageUuid) {
- notifyNamenodeBlock(block, BlockStatus.RECEIVING_BLOCK, null, storageUuid);
+ notifyNamenodeBlock(block, BlockStatus.RECEIVING_BLOCK, null, storageUuid,
+ false);
}
void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) {
- notifyNamenodeBlock(block, BlockStatus.DELETED_BLOCK, null, storageUuid);
+ notifyNamenodeBlock(block, BlockStatus.DELETED_BLOCK, null, storageUuid,
+ false);
}
private void notifyNamenodeBlock(ExtendedBlock block, BlockStatus status,
- String delHint, String storageUuid) {
+ String delHint, String storageUuid, boolean isOnTransientStorage) {
checkBlock(block);
final ReceivedDeletedBlockInfo info = new ReceivedDeletedBlockInfo(
block.getLocalBlock(), status, delHint);
final DatanodeStorage storage = dn.getFSDataset().getStorage(storageUuid);
for (BPServiceActor actor : bpServices) {
- actor.getIbrManager().notifyNamenodeBlock(info, storage);
+ actor.getIbrManager().notifyNamenodeBlock(info, storage,
+ isOnTransientStorage);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 4b987b0..d3d46be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -97,8 +97,7 @@ class BPServiceActor implements Runnable {
private final DNConf dnConf;
private long prevBlockReportId;
- private final IncrementalBlockReportManager ibrManager
- = new IncrementalBlockReportManager();
+ private final IncrementalBlockReportManager ibrManager;
private DatanodeRegistration bpRegistration;
final LinkedList<BPServiceActorAction> bpThreadQueue
@@ -109,6 +108,7 @@ class BPServiceActor implements Runnable {
this.dn = bpos.getDataNode();
this.nnAddr = nnAddr;
this.dnConf = dn.getDnConf();
+ this.ibrManager = new IncrementalBlockReportManager(dnConf.ibrInterval);
prevBlockReportId = ThreadLocalRandom.current().nextLong();
scheduler = new Scheduler(dnConf.heartBeatInterval, dnConf.blockReportInterval);
}
@@ -568,20 +568,9 @@ class BPServiceActor implements Runnable {
processCommand(new DatanodeCommand[]{ cmd });
}
- //
// There is no work to do; sleep until hearbeat timer elapses,
// or work arrives, and then iterate again.
- //
- long waitTime = scheduler.getHeartbeatWaitTime();
- synchronized(ibrManager) {
- if (waitTime > 0 && !ibrManager.sendImmediately()) {
- try {
- ibrManager.wait(waitTime);
- } catch (InterruptedException ie) {
- LOG.warn("BPOfferService for " + this + " interrupted");
- }
- }
- } // synchronized
+ ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime());
} catch(RemoteException re) {
String reClass = re.getClassName();
if (UnregisteredNodeException.class.getName().equals(reClass) ||
@@ -768,7 +757,7 @@ class BPServiceActor implements Runnable {
void triggerBlockReport(BlockReportOptions options) {
if (options.isIncremental()) {
LOG.info(bpos.toString() + ": scheduling an incremental block report.");
- ibrManager.triggerIBR();
+ ibrManager.triggerIBR(true);
} else {
LOG.info(bpos.toString() + ": scheduling a full block report.");
synchronized(ibrManager) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 0dc8cab..dfa4031 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -302,8 +302,8 @@ class BlockReceiver implements Closeable {
/** Return the datanode object. */
DataNode getDataNode() {return datanode;}
- String getStorageUuid() {
- return replicaInfo.getStorageUuid();
+ Replica getReplica() {
+ return replicaInfo;
}
/**
@@ -1439,8 +1439,8 @@ class BlockReceiver implements Closeable {
datanode.data.setPinning(block);
}
- datanode.closeBlock(
- block, DataNode.EMPTY_DEL_HINT, replicaInfo.getStorageUuid());
+ datanode.closeBlock(block, null, replicaInfo.getStorageUuid(),
+ replicaInfo.isOnTransientStorage());
if (ClientTraceLog.isInfoEnabled() && isClient) {
long offset = 0;
DatanodeRegistration dnR = datanode.getDNRegistrationForBP(block
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 0f84fc5..0fa6f6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -89,6 +89,7 @@ public class DNConf {
final long heartBeatInterval;
final long blockReportInterval;
final long blockReportSplitThreshold;
+ final long ibrInterval;
final long initialBlockReportDelayMs;
final long cacheReportInterval;
final long dfsclientSlowIoWarningThresholdMs;
@@ -156,6 +157,9 @@ public class DNConf {
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
this.blockReportInterval = conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT);
+ this.ibrInterval = conf.getLong(
+ DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY,
+ DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_DEFAULT);
this.blockReportSplitThreshold = conf.getLong(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY,
DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT);
this.cacheReportInterval = conf.getLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index b347129..470d3ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -315,7 +315,6 @@ public class DataNode extends ReconfigurableBase
volatile FsDatasetSpi<? extends FsVolumeSpi> data = null;
private String clusterId = null;
- public final static String EMPTY_DEL_HINT = "";
final AtomicInteger xmitsInProgress = new AtomicInteger();
Daemon dataXceiverServer = null;
DataXceiverServer xserver = null;
@@ -1095,11 +1094,12 @@ public class DataNode extends ReconfigurableBase
}
// calls specific to BP
- public void notifyNamenodeReceivedBlock(
- ExtendedBlock block, String delHint, String storageUuid) {
+ public void notifyNamenodeReceivedBlock(ExtendedBlock block, String delHint,
+ String storageUuid, boolean isOnTransientStorage) {
BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId());
if(bpos != null) {
- bpos.notifyNamenodeReceivedBlock(block, delHint, storageUuid);
+ bpos.notifyNamenodeReceivedBlock(block, delHint, storageUuid,
+ isOnTransientStorage);
} else {
LOG.error("Cannot find BPOfferService for reporting block received for bpid="
+ block.getBlockPoolId());
@@ -2380,15 +2380,11 @@ public class DataNode extends ReconfigurableBase
* @param delHint hint on which excess block to delete
* @param storageUuid UUID of the storage where block is stored
*/
- void closeBlock(ExtendedBlock block, String delHint, String storageUuid) {
+ void closeBlock(ExtendedBlock block, String delHint, String storageUuid,
+ boolean isTransientStorage) {
metrics.incrBlocksWritten();
- BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId());
- if(bpos != null) {
- bpos.notifyNamenodeReceivedBlock(block, delHint, storageUuid);
- } else {
- LOG.warn("Cannot find BPOfferService for reporting block received for bpid="
- + block.getBlockPoolId());
- }
+ notifyNamenodeReceivedBlock(block, delHint, storageUuid,
+ isTransientStorage);
}
/** Start a single datanode daemon and wait for it to finish.
@@ -2718,7 +2714,7 @@ public class DataNode extends ReconfigurableBase
public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock,
final long recoveryId, final long newBlockId, final long newLength)
throws IOException {
- final String storageID = data.updateReplicaUnderRecovery(oldBlock,
+ final Replica r = data.updateReplicaUnderRecovery(oldBlock,
recoveryId, newBlockId, newLength);
// Notify the namenode of the updated block info. This is important
// for HA, since otherwise the standby node may lose track of the
@@ -2727,7 +2723,9 @@ public class DataNode extends ReconfigurableBase
newBlock.setGenerationStamp(recoveryId);
newBlock.setBlockId(newBlockId);
newBlock.setNumBytes(newLength);
- notifyNamenodeReceivedBlock(newBlock, "", storageID);
+ final String storageID = r.getStorageUuid();
+ notifyNamenodeReceivedBlock(newBlock, null, storageID,
+ r.isOnTransientStorage());
return storageID;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 99eb6fa..0041cd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -672,7 +672,9 @@ class DataXceiver extends Receiver implements Runnable {
String firstBadLink = ""; // first datanode that failed in connection setup
Status mirrorInStatus = SUCCESS;
final String storageUuid;
+ final boolean isOnTransientStorage;
try {
+ final Replica replica;
if (isDatanode ||
stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) {
// open a block receiver
@@ -682,12 +684,13 @@ class DataXceiver extends Receiver implements Runnable {
stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd,
clientname, srcDataNode, datanode, requestedChecksum,
cachingStrategy, allowLazyPersist, pinning);
-
- storageUuid = blockReceiver.getStorageUuid();
+ replica = blockReceiver.getReplica();
} else {
- storageUuid = datanode.data.recoverClose(
+ replica = datanode.data.recoverClose(
block, latestGenerationStamp, minBytesRcvd);
}
+ storageUuid = replica.getStorageUuid();
+ isOnTransientStorage = replica.isOnTransientStorage();
//
// Connect to downstream machine, if appropriate
@@ -830,7 +833,7 @@ class DataXceiver extends Receiver implements Runnable {
// the block is finalized in the PacketResponder.
if (isDatanode ||
stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) {
- datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT, storageUuid);
+ datanode.closeBlock(block, null, storageUuid, isOnTransientStorage);
LOG.info("Received " + block + " src: " + remoteAddress + " dest: "
+ localAddress + " of size " + block.getNumBytes());
}
@@ -1146,8 +1149,9 @@ class DataXceiver extends Receiver implements Runnable {
dataXceiverServer.balanceThrottler, null, true);
// notify name node
+ final Replica r = blockReceiver.getReplica();
datanode.notifyNamenodeReceivedBlock(
- block, delHint, blockReceiver.getStorageUuid());
+ block, delHint, r.getStorageUuid(), r.isOnTransientStorage());
LOG.info("Moved " + block + " from " + peer.getRemoteAddressString()
+ ", delHint=" + delHint);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
index 4462f94..b9b348a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.util.Time.monotonicNow;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -33,6 +34,8 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
@@ -42,6 +45,9 @@ import com.google.common.collect.Maps;
*/
@InterfaceAudience.Private
class IncrementalBlockReportManager {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ IncrementalBlockReportManager.class);
+
private static class PerStorageIBR {
/** The blocks in this IBR. */
final Map<Block, ReceivedDeletedBlockInfo> blocks = Maps.newHashMap();
@@ -103,8 +109,29 @@ class IncrementalBlockReportManager {
*/
private volatile boolean readyToSend = false;
+ /** The time interval between two IBRs. */
+ private final long ibrInterval;
+
+ /** The timestamp of the last IBR. */
+ private volatile long lastIBR;
+
+ IncrementalBlockReportManager(final long ibrInterval) {
+ this.ibrInterval = ibrInterval;
+ this.lastIBR = monotonicNow() - ibrInterval;
+ }
+
boolean sendImmediately() {
- return readyToSend;
+ return readyToSend && monotonicNow() - ibrInterval >= lastIBR;
+ }
+
+ synchronized void waitTillNextIBR(long waitTime) {
+ if (waitTime > 0 && !sendImmediately()) {
+ try {
+ wait(ibrInterval > 0 && ibrInterval < waitTime? ibrInterval: waitTime);
+ } catch (InterruptedException ie) {
+ LOG.warn(getClass().getSimpleName() + " interrupted");
+ }
+ }
}
private synchronized StorageReceivedDeletedBlocks[] generateIBRs() {
@@ -144,6 +171,9 @@ class IncrementalBlockReportManager {
}
// Send incremental block reports to the Namenode outside the lock
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("call blockReceivedAndDeleted: " + Arrays.toString(reports));
+ }
boolean success = false;
final long startTime = monotonicNow();
try {
@@ -151,7 +181,9 @@ class IncrementalBlockReportManager {
success = true;
} finally {
metrics.addIncrementalBlockReport(monotonicNow() - startTime);
- if (!success) {
+ if (success) {
+ lastIBR = startTime;
+ } else {
// If we didn't succeed in sending the report, put all of the
// blocks back onto our queue, but only in the case where we
// didn't put something newer in the meantime.
@@ -191,7 +223,7 @@ class IncrementalBlockReportManager {
}
synchronized void notifyNamenodeBlock(ReceivedDeletedBlockInfo rdbi,
- DatanodeStorage storage) {
+ DatanodeStorage storage, boolean isOnTransientStorage) {
addRDBI(rdbi, storage);
final BlockStatus status = rdbi.getStatus();
@@ -200,18 +232,23 @@ class IncrementalBlockReportManager {
readyToSend = true;
} else if (status == BlockStatus.RECEIVED_BLOCK) {
// the report is sent right away.
- triggerIBR();
+ triggerIBR(isOnTransientStorage);
}
}
- synchronized void triggerIBR() {
+ synchronized void triggerIBR(boolean force) {
readyToSend = true;
- notifyAll();
+ if (force) {
+ lastIBR = monotonicNow() - ibrInterval;
+ }
+ if (sendImmediately()) {
+ notifyAll();
+ }
}
@VisibleForTesting
synchronized void triggerDeletionReportForTests() {
- triggerIBR();
+ triggerIBR(true);
while (sendImmediately()) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 63f0377..277b271 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -374,7 +374,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* @return the storage uuid of the replica.
* @throws IOException
*/
- String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen
+ Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen
) throws IOException;
/**
@@ -524,7 +524,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* Update replica's generation stamp and length and finalize it.
* @return the ID of storage that stores the block
*/
- String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
+ Replica updateReplicaUnderRecovery(ExtendedBlock oldBlock,
long recoveryId, long newBlockId, long newLength) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d1cb836..2e8226a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1281,7 +1281,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public synchronized String recoverClose(ExtendedBlock b, long newGS,
+ public synchronized Replica recoverClose(ExtendedBlock b, long newGS,
long expectedBlockLen) throws IOException {
LOG.info("Recover failed close " + b);
// check replica's state
@@ -1292,7 +1292,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (replicaInfo.getState() == ReplicaState.RBW) {
finalizeReplica(b.getBlockPoolId(), replicaInfo);
}
- return replicaInfo.getStorageUuid();
+ return replicaInfo;
}
/**
@@ -2424,7 +2424,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public synchronized String updateReplicaUnderRecovery(
+ public synchronized Replica updateReplicaUnderRecovery(
final ExtendedBlock oldBlock,
final long recoveryId,
final long newBlockId,
@@ -2484,8 +2484,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
//check replica files after update
checkReplicaFiles(finalized);
- //return storage ID
- return getVolume(new ExtendedBlock(bpid, finalized)).getStorageID();
+ return finalized;
}
private FinalizedReplica updateReplicaUnderRecovery(
@@ -2826,7 +2825,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
datanode.getShortCircuitRegistry().processBlockInvalidation(
ExtendedBlockId.fromExtendedBlock(extendedBlock));
datanode.notifyNamenodeReceivedBlock(
- extendedBlock, null, newReplicaInfo.getStorageUuid());
+ extendedBlock, null, newReplicaInfo.getStorageUuid(),
+ newReplicaInfo.isOnTransientStorage());
// Remove the old replicas
if (blockFile.delete() || !blockFile.exists()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
index db9505a..29216cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.protocol;
+import java.util.Arrays;
+
/**
* Report of block received and deleted per Datanode
* storage.
@@ -51,4 +53,9 @@ public class StorageReceivedDeletedBlocks {
this.storage = storage;
this.blocks = blocks;
}
+
+ @Override
+ public String toString() {
+ return storage + Arrays.toString(blocks);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index f4e88b7..1fdedca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -899,7 +899,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override // FsDatasetSpi
- public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
+ public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
throws IOException {
final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
BInfo binfo = map.get(b.getLocalBlock());
@@ -913,7 +913,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
map.remove(b.getLocalBlock());
binfo.theBlock.setGenerationStamp(newGS);
map.put(binfo.theBlock, binfo);
- return binfo.getStorageUuid();
+ return binfo;
}
@Override // FsDatasetSpi
@@ -1192,12 +1192,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override // FsDatasetSpi
- public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
+ public Replica updateReplicaUnderRecovery(ExtendedBlock oldBlock,
long recoveryId,
long newBlockId,
- long newlength) {
- // Caller does not care about the exact Storage UUID returned.
- return datanodeUuid;
+ long newlength) throws IOException {
+ return getMap(oldBlock.getBlockPoolId()).get(oldBlock.getLocalBlock());
}
@Override // FsDatasetSpi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index cb5f272..1421f0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -193,7 +193,7 @@ public class TestBPOfferService {
waitForBlockReport(mockNN2);
// When we receive a block, it should report it to both NNs
- bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, "", "");
+ bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, null, "", false);
ReceivedDeletedBlockInfo[] ret = waitForBlockReceived(FAKE_BLOCK, mockNN1);
assertEquals(1, ret.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
new file mode 100644
index 0000000..38c8a38
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.MetricsAsserts;
+import org.apache.hadoop.util.Time;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This test verifies that incremental block reports are sent in batch mode
+ * and the namenode allows closing a file with COMMITTED blocks.
+ */
+public class TestBatchIbr {
+ public static final Log LOG = LogFactory.getLog(TestBatchIbr.class);
+
+ private static final short NUM_DATANODES = 4;
+ private static final int BLOCK_SIZE = 1024;
+ private static final int MAX_BLOCK_NUM = 8;
+ private static final int NUM_FILES = 1000;
+ private static final int NUM_THREADS = 128;
+
+ private static final ThreadLocalBuffer IO_BUF = new ThreadLocalBuffer();
+ private static final ThreadLocalBuffer VERIFY_BUF = new ThreadLocalBuffer();
+
+ static {
+ GenericTestUtils.setLogLevel(
+ LogFactory.getLog(IncrementalBlockReportManager.class), Level.ALL);
+ }
+
+ static HdfsConfiguration newConf(long ibrInterval) throws IOException {
+ final HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setLong(DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setBoolean(ReplaceDatanodeOnFailure.BEST_EFFORT_KEY, true);
+
+ if (ibrInterval > 0) {
+ conf.setLong(DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY, ibrInterval);
+ }
+ return conf;
+ }
+
+ static ExecutorService createExecutor() throws Exception {
+ final ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS);
+ final ExecutorCompletionService<Path> completion
+ = new ExecutorCompletionService<>(executor);
+
+ // initialize all threads and buffers
+ for(int i = 0; i < NUM_THREADS; i++) {
+ completion.submit(new Callable<Path>() {
+ @Override
+ public Path call() throws Exception {
+ IO_BUF.get();
+ VERIFY_BUF.get();
+ return null;
+ }
+ });
+ }
+ for(int i = 0; i < NUM_THREADS; i++) {
+ completion.take().get();
+ }
+ return executor;
+ }
+
+ static void runIbrTest(final long ibrInterval) throws Exception {
+ final ExecutorService executor = createExecutor();
+ final Random ran = new Random();
+
+ final Configuration conf = newConf(ibrInterval);
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_DATANODES).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+
+ try {
+ final String dirPathString = "/dir";
+ final Path dir = new Path(dirPathString);
+ dfs.mkdirs(dir);
+
+ // start testing
+ final long testStartTime = Time.monotonicNow();
+ final ExecutorCompletionService<Path> createService
+ = new ExecutorCompletionService<>(executor);
+ final AtomicLong createFileTime = new AtomicLong();
+ final AtomicInteger numBlockCreated = new AtomicInteger();
+
+ // create files
+ for(int i = 0; i < NUM_FILES; i++) {
+ createService.submit(new Callable<Path>() {
+ @Override
+ public Path call() throws Exception {
+ final long start = Time.monotonicNow();
+ try {
+ final long seed = ran.nextLong();
+ final int numBlocks = ran.nextInt(MAX_BLOCK_NUM) + 1;
+ numBlockCreated.addAndGet(numBlocks);
+ return createFile(dir, numBlocks, seed, dfs);
+ } finally {
+ createFileTime.addAndGet(Time.monotonicNow() - start);
+ }
+ }
+ });
+ }
+
+ // verify files
+ final ExecutorCompletionService<Boolean> verifyService
+ = new ExecutorCompletionService<>(executor);
+ final AtomicLong verifyFileTime = new AtomicLong();
+ for(int i = 0; i < NUM_FILES; i++) {
+ final Path file = createService.take().get();
+ verifyService.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ final long start = Time.monotonicNow();
+ try {
+ return verifyFile(file, dfs);
+ } finally {
+ verifyFileTime.addAndGet(Time.monotonicNow() - start);
+ }
+ }
+ });
+ }
+ for(int i = 0; i < NUM_FILES; i++) {
+ Assert.assertTrue(verifyService.take().get());
+ }
+ final long testEndTime = Time.monotonicNow();
+
+ LOG.info("ibrInterval=" + ibrInterval + " ("
+ + toConfString(DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY, conf)
+ + "), numBlockCreated=" + numBlockCreated);
+ LOG.info("duration=" + toSecondString(testEndTime - testStartTime)
+ + ", createFileTime=" + toSecondString(createFileTime.get())
+ + ", verifyFileTime=" + toSecondString(verifyFileTime.get()));
+ LOG.info("NUM_FILES=" + NUM_FILES
+ + ", MAX_BLOCK_NUM=" + MAX_BLOCK_NUM
+ + ", BLOCK_SIZE=" + BLOCK_SIZE
+ + ", NUM_THREADS=" + NUM_THREADS
+ + ", NUM_DATANODES=" + NUM_DATANODES);
+ logIbrCounts(cluster.getDataNodes());
+ } finally {
+ executor.shutdown();
+ cluster.shutdown();
+ }
+ }
+
+ static String toConfString(String key, Configuration conf) {
+ return key + "=" + conf.get(key);
+ }
+
+ static String toSecondString(long ms) {
+ return (ms/1000.0) + "s";
+ }
+
+ static void logIbrCounts(List<DataNode> datanodes) {
+ final String name = "IncrementalBlockReportsNumOps";
+ for(DataNode dn : datanodes) {
+ final MetricsRecordBuilder m = MetricsAsserts.getMetrics(
+ dn.getMetrics().name());
+ final long ibr = MetricsAsserts.getLongCounter(name, m);
+ LOG.info(dn.getDisplayName() + ": " + name + "=" + ibr);
+ }
+
+ }
+
+ static class ThreadLocalBuffer extends ThreadLocal<byte[]> {
+ @Override
+ protected byte[] initialValue() {
+ return new byte[BLOCK_SIZE];
+ }
+ }
+
+ static byte[] nextBytes(int blockIndex, long seed, byte[] bytes) {
+ byte b = (byte)(seed ^ (seed >> blockIndex));
+ for(int i = 0; i < bytes.length; i++) {
+ bytes[i] = b++;
+ }
+ return bytes;
+ }
+
+ static Path createFile(Path dir, int numBlocks, long seed,
+ DistributedFileSystem dfs) throws IOException {
+ final Path f = new Path(dir, seed + "_" + numBlocks);
+ final byte[] bytes = IO_BUF.get();
+
+ try(FSDataOutputStream out = dfs.create(f)) {
+ for(int i = 0; i < numBlocks; i++) {
+ out.write(nextBytes(i, seed, bytes));
+ }
+ }
+ return f;
+ }
+
+ static boolean verifyFile(Path f, DistributedFileSystem dfs) {
+ final long seed;
+ final int numBlocks;
+ {
+ final String name = f.getName();
+ final int i = name.indexOf('_');
+ seed = Long.parseLong(name.substring(0, i));
+ numBlocks = Integer.parseInt(name.substring(i + 1));
+ }
+
+ final byte[] computed = IO_BUF.get();
+ final byte[] expected = VERIFY_BUF.get();
+
+ try(FSDataInputStream in = dfs.open(f)) {
+ for(int i = 0; i < numBlocks; i++) {
+ in.read(computed);
+ nextBytes(i, seed, expected);
+ Assert.assertArrayEquals(expected, computed);
+ }
+ return true;
+ } catch(Exception e) {
+ LOG.error("Failed to verify file " + f);
+ return false;
+ }
+ }
+
+ @Test
+ public void testIbr() throws Exception {
+ runIbrTest(0L);
+ runIbrTest(100L);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
index 3af959c..c21cc86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
@@ -138,12 +138,14 @@ public class TestDataXceiverLazyPersistHint {
PeerLocality locality,
NonLocalLazyPersist nonLocalLazyPersist,
final ArgumentCaptor<Boolean> captor) throws IOException {
+ final BlockReceiver mockBlockReceiver = mock(BlockReceiver.class);
+ doReturn(mock(Replica.class)).when(mockBlockReceiver).getReplica();
+
DataXceiver xceiverSpy = spy(DataXceiver.create(
getMockPeer(locality),
getMockDn(nonLocalLazyPersist),
mock(DataXceiverServer.class)));
-
- doReturn(mock(BlockReceiver.class)).when(xceiverSpy).getBlockReceiver(
+ doReturn(mockBlockReceiver).when(xceiverSpy).getBlockReceiver(
any(ExtendedBlock.class), any(StorageType.class),
any(DataInputStream.class), anyString(), anyString(),
any(BlockConstructionStage.class), anyLong(), anyLong(), anyLong(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
index 676d855..2288974 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
@@ -88,7 +88,7 @@ public class TestIncrementalBlockReports {
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(
getDummyBlock(), BlockStatus.RECEIVED_BLOCK, null);
DatanodeStorage s = singletonDn.getFSDataset().getStorage(storageUuid);
- actor.getIbrManager().notifyNamenodeBlock(rdbi, s);
+ actor.getIbrManager().notifyNamenodeBlock(rdbi, s, false);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index e3be5fd..c872e61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -176,7 +176,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
}
@Override
- public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
+ public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlkLen)
throws IOException {
return null;
}
@@ -271,7 +271,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
}
@Override
- public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
+ public Replica updateReplicaUnderRecovery(ExtendedBlock oldBlock,
long recoveryId, long newBlockId, long newLength) throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1d4e166/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
index b7efdf6..c054641 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
@@ -328,7 +328,6 @@ public class TestInterDatanodeProtocol {
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
- String bpid = cluster.getNamesystem().getBlockPoolId();
//create a file
DistributedFileSystem dfs = cluster.getFileSystem();
@@ -379,10 +378,11 @@ public class TestInterDatanodeProtocol {
}
//update
- final String storageID = fsdataset.updateReplicaUnderRecovery(
+ final Replica r = fsdataset.updateReplicaUnderRecovery(
new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid,
rri.getBlockId(), newlength);
- assertTrue(storageID != null);
+ assertTrue(r != null);
+ assertTrue(r.getStorageUuid() != null);
} finally {
if (cluster != null) cluster.shutdown();
[04/50] [abbrv] hadoop git commit: HADOOP-12825. Log slow name
resolutions. (Sidharta Seethana via stevel)
Posted by ar...@apache.org.
HADOOP-12825. Log slow name resolutions. (Sidharta Seethana via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03cfb454
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03cfb454
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03cfb454
Branch: refs/heads/HDFS-1312
Commit: 03cfb454fe5a1351e283e4678ad1b432ed231485
Parents: 798babf
Author: Steve Loughran <st...@apache.org>
Authored: Sat Feb 27 20:05:35 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Feb 27 20:23:51 2016 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/fs/CommonConfigurationKeys.java | 14 ++++++-
.../apache/hadoop/security/SecurityUtil.java | 42 +++++++++++++++++++-
.../src/main/resources/core-default.xml | 18 +++++++++
4 files changed, 74 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03cfb454/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 79cf4d5..b8ae8df 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1160,6 +1160,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12824. Collect network and disk usage on the node running Windows.
(Inigo Goiri via xyao)
+ HADOOP-12825. Log slow name resolutions.
+ (Sidharta Seethana via stevel)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03cfb454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index d3ae6b1..9b4069a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -203,7 +203,19 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"hadoop.security.token.service.use_ip";
public static final boolean HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT =
true;
-
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ public static final String HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY =
+ "hadoop.security.dns.log-slow-lookups.enabled";
+ public static final boolean
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT = false;
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ public static final String
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY =
+ "hadoop.security.dns.log-slow-lookups.threshold.ms";
+ public static final int
+ HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT = 1000;
+
/**
* HA health monitor and failover controller.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03cfb454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 38096ab..61cd516 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -30,6 +30,7 @@ import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.List;
import java.util.ServiceLoader;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -47,6 +48,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.util.StopWatch;
import org.apache.hadoop.util.StringUtils;
@@ -79,6 +81,9 @@ public class SecurityUtil {
setTokenServiceUseIp(useIp);
}
+ private static boolean logSlowLookups = getLogSlowLookupsEnabled();
+ private static int slowLookupThresholdMs = getSlowLookupThresholdMs();
+
/**
* For use only by tests and initialization
*/
@@ -480,9 +485,27 @@ public class SecurityUtil {
}
}
+ private static boolean getLogSlowLookupsEnabled() {
+ Configuration conf = new Configuration();
+
+ return conf.getBoolean(CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY,
+ CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT);
+ }
+
+ private static int getSlowLookupThresholdMs() {
+ Configuration conf = new Configuration();
+
+ return conf.getInt(CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY,
+ CommonConfigurationKeys
+ .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT);
+ }
+
/**
* Resolves a host subject to the security requirements determined by
- * hadoop.security.token.service.use_ip.
+ * hadoop.security.token.service.use_ip. Optionally logs slow resolutions.
*
* @param hostname host or ip to resolve
* @return a resolved host
@@ -491,7 +514,22 @@ public class SecurityUtil {
@InterfaceAudience.Private
public static
InetAddress getByName(String hostname) throws UnknownHostException {
- return hostResolver.getByName(hostname);
+ if (logSlowLookups || LOG.isTraceEnabled()) {
+ StopWatch lookupTimer = new StopWatch().start();
+ InetAddress result = hostResolver.getByName(hostname);
+ long elapsedMs = lookupTimer.stop().now(TimeUnit.MILLISECONDS);
+
+ if (elapsedMs >= slowLookupThresholdMs) {
+ LOG.warn("Slow name lookup for " + hostname + ". Took " + elapsedMs +
+ " ms.");
+ } else if (LOG.isTraceEnabled()) {
+ LOG.trace("Name lookup for " + hostname + " took " + elapsedMs +
+ " ms.");
+ }
+ return result;
+ } else {
+ return hostResolver.getByName(hostname);
+ }
}
interface HostResolver {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03cfb454/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 95140bf..57f5274 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -113,6 +113,24 @@
</description>
</property>
+<property>
+ <name>hadoop.security.dns.log-slow-lookups.enabled</name>
+ <value>false</value>
+ <description>
+ Time name lookups (via SecurityUtil) and log them if they exceed the
+ configured threshold.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.dns.log-slow-lookups.threshold.ms</name>
+ <value>1000</value>
+ <description>
+ If slow lookup logging is enabled, this threshold is used to decide if a
+ lookup is considered slow enough to be logged.
+ </description>
+</property>
+
<!--
=== Multiple group mapping providers configuration sample ===
This sample illustrates a typical use case for CompositeGroupsMapping where
[11/50] [abbrv] hadoop git commit: YARN-4566. Fix test failure in
TestMiniYarnClusterNodeUtilization. (Takashi Ohnishi via rohithsharmaks)
Posted by ar...@apache.org.
YARN-4566. Fix test failure in TestMiniYarnClusterNodeUtilization. (Takashi Ohnishi via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0b14f26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0b14f26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0b14f26
Branch: refs/heads/HDFS-1312
Commit: e0b14f26f5201a149218276469434df511697acc
Parents: 321a80c
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Feb 29 10:38:14 2016 +0800
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Feb 29 10:38:14 2016 +0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java | 6 ++++--
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0b14f26/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d2f871a..3c91fbd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -236,6 +236,9 @@ Release 2.9.0 - UNRELEASED
YARN-4701. When task logs are not available, port 8041 is referenced
instead of port 8042 (haibochen via rkanter)
+ YARN-4566. Fix test failure in TestMiniYarnClusterNodeUtilization.
+ (Takashi Ohnishi via rohithsharmaks)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0b14f26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
index cf83e67..49a82c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
@@ -214,8 +214,10 @@ public class TestMiniYarnClusterNodeUtilization {
// We check if the nodeUtilization is up to date
for (int i=0; i<100; i++) {
for (RMNode ni : rmContext.getRMNodes().values()) {
- if (ni.getNodeUtilization().equals(nodeUtilization)) {
- break;
+ if (ni.getNodeUtilization() != null) {
+ if (ni.getNodeUtilization().equals(nodeUtilization)) {
+ break;
+ }
}
}
Thread.sleep(100);
[25/50] [abbrv] hadoop git commit: HDFS-9733. Refactor
DFSClient#getFileChecksum and DataXceiver#blockChecksum. Contributed by Kai
Zheng
Posted by ar...@apache.org.
HDFS-9733. Refactor DFSClient#getFileChecksum and DataXceiver#blockChecksum. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/307ec80a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/307ec80a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/307ec80a
Branch: refs/heads/HDFS-1312
Commit: 307ec80acae3b4a41d21b2d4b3a55032e55fcdc6
Parents: 680f3fc
Author: Uma Maheswara Rao G <um...@intel.com>
Authored: Mon Feb 29 21:52:20 2016 -0800
Committer: Uma Maheswara Rao G <um...@intel.com>
Committed: Mon Feb 29 21:52:20 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/io/IOUtils.java | 4 +-
.../main/java/org/apache/hadoop/io/MD5Hash.java | 11 +
.../org/apache/hadoop/util/DataChecksum.java | 12 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 237 ++---------
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 49 +++
.../apache/hadoop/hdfs/FileChecksumHelper.java | 416 +++++++++++++++++++
.../protocol/datatransfer/IOStreamPair.java | 11 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/datanode/BlockChecksumHelper.java | 254 +++++++++++
.../hdfs/server/datanode/BlockSender.java | 1 -
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +
.../hdfs/server/datanode/DataXceiver.java | 162 +++-----
12 files changed, 846 insertions(+), 324 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
index 451163c..2588bf1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
@@ -261,7 +261,9 @@ public class IOUtils {
* @param stream the Stream to close
*/
public static void closeStream(java.io.Closeable stream) {
- cleanup(null, stream);
+ if (stream != null) {
+ cleanup(null, stream);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
index 822e089..aaf3ea1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
@@ -128,6 +128,17 @@ public class MD5Hash implements WritableComparable<MD5Hash> {
return new MD5Hash(digest);
}
+ /** Construct a hash value for an array of byte array. */
+ public static MD5Hash digest(byte[][] dataArr, int start, int len) {
+ byte[] digest;
+ MessageDigest digester = getDigester();
+ for (byte[] data : dataArr) {
+ digester.update(data, start, len);
+ }
+ digest = digester.digest();
+ return new MD5Hash(digest);
+ }
+
/** Construct a hash value for a String. */
public static MD5Hash digest(String string) {
return digest(UTF8.getBytes(string));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index faac587..e44b64d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -45,7 +45,7 @@ public class DataChecksum implements Checksum {
public static final int CHECKSUM_MIXED = 4;
/** The checksum types */
- public static enum Type {
+ public enum Type {
NULL (CHECKSUM_NULL, 0),
CRC32 (CHECKSUM_CRC32, 4),
CRC32C(CHECKSUM_CRC32C, 4),
@@ -55,7 +55,7 @@ public class DataChecksum implements Checksum {
public final int id;
public final int size;
- private Type(int id, int size) {
+ Type(int id, int size) {
this.id = id;
this.size = size;
}
@@ -230,17 +230,21 @@ public class DataChecksum implements Checksum {
public Type getChecksumType() {
return type;
}
+
/** @return the size for a checksum. */
public int getChecksumSize() {
return type.size;
}
+
/** @return the required checksum size given the data length. */
public int getChecksumSize(int dataSize) {
return ((dataSize - 1)/getBytesPerChecksum() + 1) * getChecksumSize();
}
+
public int getBytesPerChecksum() {
return bytesPerChecksum;
}
+
public int getNumBytesInSum() {
return inSum;
}
@@ -249,16 +253,19 @@ public class DataChecksum implements Checksum {
static public int getChecksumHeaderSize() {
return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
}
+
//Checksum Interface. Just a wrapper around member summer.
@Override
public long getValue() {
return summer.getValue();
}
+
@Override
public void reset() {
summer.reset();
inSum = 0;
}
+
@Override
public void update( byte[] b, int off, int len ) {
if ( len > 0 ) {
@@ -266,6 +273,7 @@ public class DataChecksum implements Checksum {
inSum += len;
}
}
+
@Override
public void update( int b ) {
summer.update( b );
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 15a49f1..da3f745 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -27,12 +27,9 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_LOCA
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -80,9 +77,7 @@ import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.FsTracer;
import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.InvalidPathException;
-import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
-import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
@@ -138,7 +133,6 @@ import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
-import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
@@ -146,20 +140,16 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactor
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
-import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.util.IOUtilsClient;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
import org.apache.hadoop.ipc.RPC;
@@ -1293,7 +1283,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/**
* Invoke namenode append RPC.
- * It retries in case of {@link BlockNotYetCompleteException}.
+ * It retries in case of some {@link RetriableException}.
*/
private LastBlockWithStatus callAppend(String src,
EnumSetWritable<CreateFlag> flag) throws IOException {
@@ -1695,7 +1685,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
- * Get the checksum of the whole file of a range of the file. Note that the
+ * Get the checksum of the whole file or a range of the file. Note that the
* range always starts from the beginning of the file.
* @param src The file path
* @param length the length of the range, i.e., the range is [0, length]
@@ -1706,9 +1696,23 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
throws IOException {
checkOpen();
Preconditions.checkArgument(length >= 0);
+
+ LocatedBlocks blockLocations = getBlockLocations(src, length);
+
+ FileChecksumHelper.FileChecksumComputer maker =
+ new FileChecksumHelper.ReplicatedFileChecksumComputer(src, length,
+ blockLocations, namenode, this);
+
+ maker.compute();
+
+ return maker.getFileChecksum();
+ }
+
+ protected LocatedBlocks getBlockLocations(String src,
+ long length) throws IOException {
//get block locations for the file range
- LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0,
- length);
+ LocatedBlocks blockLocations = callGetBlockLocations(namenode,
+ src, 0, length);
if (null == blockLocations) {
throw new FileNotFoundException("File does not exist: " + src);
}
@@ -1716,194 +1720,15 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
throw new IOException("Fail to get checksum, since file " + src
+ " is under construction.");
}
- List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
- final DataOutputBuffer md5out = new DataOutputBuffer();
- int bytesPerCRC = -1;
- DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
- long crcPerBlock = 0;
- boolean refetchBlocks = false;
- int lastRetriedIndex = -1;
-
- // get block checksum for each block
- long remaining = length;
- if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
- remaining = Math.min(length, blockLocations.getFileLength());
- }
- for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
- if (refetchBlocks) { // refetch to get fresh tokens
- blockLocations = callGetBlockLocations(namenode, src, 0, length);
- if (null == blockLocations) {
- throw new FileNotFoundException("File does not exist: " + src);
- }
- if (blockLocations.isUnderConstruction()) {
- throw new IOException("Fail to get checksum, since file " + src
- + " is under construction.");
- }
- locatedblocks = blockLocations.getLocatedBlocks();
- refetchBlocks = false;
- }
- LocatedBlock lb = locatedblocks.get(i);
- final ExtendedBlock block = lb.getBlock();
- if (remaining < block.getNumBytes()) {
- block.setNumBytes(remaining);
- }
- remaining -= block.getNumBytes();
- final DatanodeInfo[] datanodes = lb.getLocations();
-
- //try each datanode location of the block
- final int timeout = 3000 * datanodes.length +
- dfsClientConf.getSocketTimeout();
- boolean done = false;
- for(int j = 0; !done && j < datanodes.length; j++) {
- DataOutputStream out = null;
- DataInputStream in = null;
-
- try {
- //connect to a datanode
- IOStreamPair pair = connectToDN(datanodes[j], timeout, lb);
- out = new DataOutputStream(new BufferedOutputStream(pair.out,
- smallBufferSize));
- in = new DataInputStream(pair.in);
-
- LOG.debug("write to {}: {}, block={}",
- datanodes[j], Op.BLOCK_CHECKSUM, block);
- // get block MD5
- new Sender(out).blockChecksum(block, lb.getBlockToken());
-
- final BlockOpResponseProto reply =
- BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(in));
-
- String logInfo = "for block " + block + " from datanode " +
- datanodes[j];
- DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);
-
- OpBlockChecksumResponseProto checksumData =
- reply.getChecksumResponse();
-
- //read byte-per-checksum
- final int bpc = checksumData.getBytesPerCrc();
- if (i == 0) { //first block
- bytesPerCRC = bpc;
- }
- else if (bpc != bytesPerCRC) {
- throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
- + " but bytesPerCRC=" + bytesPerCRC);
- }
-
- //read crc-per-block
- final long cpb = checksumData.getCrcPerBlock();
- if (locatedblocks.size() > 1 && i == 0) {
- crcPerBlock = cpb;
- }
-
- //read md5
- final MD5Hash md5 = new MD5Hash(
- checksumData.getMd5().toByteArray());
- md5.write(md5out);
-
- // read crc-type
- final DataChecksum.Type ct;
- if (checksumData.hasCrcType()) {
- ct = PBHelperClient.convert(checksumData
- .getCrcType());
- } else {
- LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
- "inferring checksum by reading first byte");
- ct = inferChecksumTypeByReading(lb, datanodes[j]);
- }
-
- if (i == 0) { // first block
- crcType = ct;
- } else if (crcType != DataChecksum.Type.MIXED
- && crcType != ct) {
- // if crc types are mixed in a file
- crcType = DataChecksum.Type.MIXED;
- }
-
- done = true;
-
- if (LOG.isDebugEnabled()) {
- if (i == 0) {
- LOG.debug("set bytesPerCRC=" + bytesPerCRC
- + ", crcPerBlock=" + crcPerBlock);
- }
- LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
- }
- } catch (InvalidBlockTokenException ibte) {
- if (i > lastRetriedIndex) {
- LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
- + "for file {} for block {} from datanode {}. Will retry "
- + "the block once.",
- src, block, datanodes[j]);
- lastRetriedIndex = i;
- done = true; // actually it's not done; but we'll retry
- i--; // repeat at i-th block
- refetchBlocks = true;
- break;
- }
- } catch (IOException ie) {
- LOG.warn("src=" + src + ", datanodes["+j+"]=" + datanodes[j], ie);
- } finally {
- IOUtils.closeStream(in);
- IOUtils.closeStream(out);
- }
- }
-
- if (!done) {
- throw new IOException("Fail to get block MD5 for " + block);
- }
- }
- //compute file MD5
- final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
- switch (crcType) {
- case CRC32:
- return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
- crcPerBlock, fileMD5);
- case CRC32C:
- return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC,
- crcPerBlock, fileMD5);
- default:
- // If there is no block allocated for the file,
- // return one with the magic entry that matches what previous
- // hdfs versions return.
- if (locatedblocks.size() == 0) {
- return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
- }
-
- // we should never get here since the validity was checked
- // when getCrcType() was called above.
- return null;
- }
+ return blockLocations;
}
- /**
- * Connect to the given datanode's datantrasfer port, and return
- * the resulting IOStreamPair. This includes encryption wrapping, etc.
- */
- private IOStreamPair connectToDN(DatanodeInfo dn, int timeout,
- LocatedBlock lb) throws IOException {
- boolean success = false;
- Socket sock = null;
- try {
- sock = socketFactory.createSocket();
- String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname());
- LOG.debug("Connecting to datanode {}", dnAddr);
- NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
- sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay());
- sock.setSoTimeout(timeout);
-
- OutputStream unbufOut = NetUtils.getOutputStream(sock);
- InputStream unbufIn = NetUtils.getInputStream(sock);
- IOStreamPair ret = saslClient.newSocketSend(sock, unbufOut, unbufIn, this,
- lb.getBlockToken(), dn);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeSocket(sock);
- }
- }
+ protected IOStreamPair connectToDN(DatanodeInfo dn, int timeout,
+ Token<BlockTokenIdentifier> blockToken)
+ throws IOException {
+ return DFSUtilClient.connectToDN(dn, timeout, conf, saslClient,
+ socketFactory, getConf().isConnectToDnViaHostname(), this, blockToken);
}
/**
@@ -1917,19 +1742,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @return the inferred checksum type
* @throws IOException if an error occurs
*/
- private Type inferChecksumTypeByReading(LocatedBlock lb, DatanodeInfo dn)
+ protected Type inferChecksumTypeByReading(LocatedBlock lb, DatanodeInfo dn)
throws IOException {
- IOStreamPair pair = connectToDN(dn, dfsClientConf.getSocketTimeout(), lb);
+ IOStreamPair pair = connectToDN(dn, dfsClientConf.getSocketTimeout(),
+ lb.getBlockToken());
try {
- DataOutputStream out = new DataOutputStream(
- new BufferedOutputStream(pair.out, smallBufferSize));
- DataInputStream in = new DataInputStream(pair.in);
-
- new Sender(out).readBlock(lb.getBlock(), lb.getBlockToken(), clientName,
+ new Sender((DataOutputStream) pair.out).readBlock(lb.getBlock(),
+ lb.getBlockToken(), clientName,
0, 1, true, CachingStrategy.newDefaultStrategy());
final BlockOpResponseProto reply =
- BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(in));
+ BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(pair.in));
String logInfo = "trying to read " + lb.getBlock() + " from datanode " +
dn;
DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index d646252..880234e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.hdfs.protocol.ReconfigurationProtocol;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.protocolPB.ReconfigurationProtocolTranslatorPB;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.util.IOUtilsClient;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
@@ -56,8 +58,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.SocketFactory;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
+import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -717,4 +724,46 @@ public class DFSUtilClient {
return corruptionMap;
}
}
+
+ /**
+ * Connect to the given datanode's datantrasfer port, and return
+ * the resulting IOStreamPair. This includes encryption wrapping, etc.
+ */
+ public static IOStreamPair connectToDN(DatanodeInfo dn, int timeout,
+ Configuration conf,
+ SaslDataTransferClient saslClient,
+ SocketFactory socketFactory,
+ boolean connectToDnViaHostname,
+ DataEncryptionKeyFactory dekFactory,
+ Token<BlockTokenIdentifier> blockToken)
+ throws IOException {
+
+ boolean success = false;
+ Socket sock = null;
+ try {
+ sock = socketFactory.createSocket();
+ String dnAddr = dn.getXferAddr(connectToDnViaHostname);
+ LOG.debug("Connecting to datanode {}", dnAddr);
+ NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
+ sock.setSoTimeout(timeout);
+
+ OutputStream unbufOut = NetUtils.getOutputStream(sock);
+ InputStream unbufIn = NetUtils.getInputStream(sock);
+ IOStreamPair pair = saslClient.newSocketSend(sock, unbufOut,
+ unbufIn, dekFactory, blockToken, dn);
+
+ IOStreamPair result = new IOStreamPair(
+ new DataInputStream(pair.in),
+ new DataOutputStream(new BufferedOutputStream(pair.out,
+ DFSUtilClient.getSmallBufferSize(conf)))
+ );
+
+ success = true;
+ return result;
+ } finally {
+ if (!success) {
+ IOUtils.closeSocket(sock);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
new file mode 100644
index 0000000..d15db9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -0,0 +1,416 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
+import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
+import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
+import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.DataChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Utility classes to compute file checksum for both replicated and striped
+ * files.
+ */
+final class FileChecksumHelper {
+ static final Logger LOG =
+ LoggerFactory.getLogger(FileChecksumHelper.class);
+
+ private FileChecksumHelper() {}
+
+ /**
+ * A common abstract class to compute file checksum.
+ */
+ static abstract class FileChecksumComputer {
+ private final String src;
+ private final long length;
+ private final DFSClient client;
+ private final ClientProtocol namenode;
+ private final DataOutputBuffer md5out = new DataOutputBuffer();
+
+ private MD5MD5CRC32FileChecksum fileChecksum;
+ private LocatedBlocks blockLocations;
+
+ private int timeout;
+ private List<LocatedBlock> locatedBlocks;
+ private long remaining = 0L;
+
+ private int bytesPerCRC = -1;
+ private DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
+ private long crcPerBlock = 0;
+ private boolean refetchBlocks = false;
+ private int lastRetriedIndex = -1;
+
+ /**
+ * Constructor that accepts all the input parameters for the computing.
+ */
+ FileChecksumComputer(String src, long length,
+ LocatedBlocks blockLocations,
+ ClientProtocol namenode,
+ DFSClient client) throws IOException {
+ this.src = src;
+ this.length = length;
+ this.blockLocations = blockLocations;
+ this.namenode = namenode;
+ this.client = client;
+
+ this.remaining = length;
+ if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+ this.remaining = Math.min(length, blockLocations.getFileLength());
+ }
+
+ this.locatedBlocks = blockLocations.getLocatedBlocks();
+ }
+
+ String getSrc() {
+ return src;
+ }
+
+ long getLength() {
+ return length;
+ }
+
+ DFSClient getClient() {
+ return client;
+ }
+
+ ClientProtocol getNamenode() {
+ return namenode;
+ }
+
+ DataOutputBuffer getMd5out() {
+ return md5out;
+ }
+
+ MD5MD5CRC32FileChecksum getFileChecksum() {
+ return fileChecksum;
+ }
+
+ LocatedBlocks getBlockLocations() {
+ return blockLocations;
+ }
+
+ void setBlockLocations(LocatedBlocks blockLocations) {
+ this.blockLocations = blockLocations;
+ }
+
+ int getTimeout() {
+ return timeout;
+ }
+
+ void setTimeout(int timeout) {
+ this.timeout = timeout;
+ }
+
+ List<LocatedBlock> getLocatedBlocks() {
+ return locatedBlocks;
+ }
+
+ void setLocatedBlocks(List<LocatedBlock> locatedBlocks) {
+ this.locatedBlocks = locatedBlocks;
+ }
+
+ long getRemaining() {
+ return remaining;
+ }
+
+ void setRemaining(long remaining) {
+ this.remaining = remaining;
+ }
+
+ int getBytesPerCRC() {
+ return bytesPerCRC;
+ }
+
+ void setBytesPerCRC(int bytesPerCRC) {
+ this.bytesPerCRC = bytesPerCRC;
+ }
+
+ DataChecksum.Type getCrcType() {
+ return crcType;
+ }
+
+ void setCrcType(DataChecksum.Type crcType) {
+ this.crcType = crcType;
+ }
+
+ long getCrcPerBlock() {
+ return crcPerBlock;
+ }
+
+ void setCrcPerBlock(long crcPerBlock) {
+ this.crcPerBlock = crcPerBlock;
+ }
+
+ boolean isRefetchBlocks() {
+ return refetchBlocks;
+ }
+
+ void setRefetchBlocks(boolean refetchBlocks) {
+ this.refetchBlocks = refetchBlocks;
+ }
+
+ int getLastRetriedIndex() {
+ return lastRetriedIndex;
+ }
+
+ void setLastRetriedIndex(int lastRetriedIndex) {
+ this.lastRetriedIndex = lastRetriedIndex;
+ }
+
+ /**
+ * Perform the file checksum computing. The intermediate results are stored
+ * in the object and will be used later.
+ * @throws IOException
+ */
+ void compute() throws IOException {
+ checksumBlocks();
+
+ fileChecksum = makeFinalResult();
+ }
+
+ /**
+ * Compute and aggregate block checksums block by block.
+ * @throws IOException
+ */
+ abstract void checksumBlocks() throws IOException;
+
+ /**
+ * Make final file checksum result given the computing process done.
+ */
+ MD5MD5CRC32FileChecksum makeFinalResult() {
+ //compute file MD5
+ final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
+ switch (crcType) {
+ case CRC32:
+ return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
+ crcPerBlock, fileMD5);
+ case CRC32C:
+ return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC,
+ crcPerBlock, fileMD5);
+ default:
+ // If there is no block allocated for the file,
+ // return one with the magic entry that matches what previous
+ // hdfs versions return.
+ if (locatedBlocks.isEmpty()) {
+ return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
+ }
+
+ // we should never get here since the validity was checked
+ // when getCrcType() was called above.
+ return null;
+ }
+ }
+
+ /**
+ * Create and return a sender given an IO stream pair.
+ */
+ Sender createSender(IOStreamPair pair) {
+ DataOutputStream out = (DataOutputStream) pair.out;
+ return new Sender(out);
+ }
+
+ /**
+ * Close an IO stream pair.
+ */
+ void close(IOStreamPair pair) {
+ if (pair != null) {
+ IOUtils.closeStream(pair.in);
+ IOUtils.closeStream(pair.out);
+ }
+ }
+ }
+
+ /**
+ * Replicated file checksum computer.
+ */
+ static class ReplicatedFileChecksumComputer extends FileChecksumComputer {
+ private int blockIdx;
+
+ ReplicatedFileChecksumComputer(String src, long length,
+ LocatedBlocks blockLocations,
+ ClientProtocol namenode,
+ DFSClient client) throws IOException {
+ super(src, length, blockLocations, namenode, client);
+ }
+
+ @Override
+ void checksumBlocks() throws IOException {
+ // get block checksum for each block
+ for (blockIdx = 0;
+ blockIdx < getLocatedBlocks().size() && getRemaining() >= 0;
+ blockIdx++) {
+ if (isRefetchBlocks()) { // refetch to get fresh tokens
+ setBlockLocations(getClient().getBlockLocations(getSrc(),
+ getLength()));
+ setLocatedBlocks(getBlockLocations().getLocatedBlocks());
+ setRefetchBlocks(false);
+ }
+
+ LocatedBlock locatedBlock = getLocatedBlocks().get(blockIdx);
+
+ if (!checksumBlock(locatedBlock)) {
+ throw new IOException("Fail to get block MD5 for " + locatedBlock);
+ }
+ }
+ }
+
+ /**
+ * Return true when sounds good to continue or retry, false when severe
+ * condition or totally failed.
+ */
+ private boolean checksumBlock(
+ LocatedBlock locatedBlock) throws IOException {
+ ExtendedBlock block = locatedBlock.getBlock();
+ if (getRemaining() < block.getNumBytes()) {
+ block.setNumBytes(getRemaining());
+ }
+ setRemaining(getRemaining() - block.getNumBytes());
+
+ DatanodeInfo[] datanodes = locatedBlock.getLocations();
+
+ int tmpTimeout = 3000 * datanodes.length +
+ getClient().getConf().getSocketTimeout();
+ setTimeout(tmpTimeout);
+
+ //try each datanode location of the block
+ boolean done = false;
+ for (int j = 0; !done && j < datanodes.length; j++) {
+ try {
+ tryDatanode(locatedBlock, datanodes[j]);
+ done = true;
+ } catch (InvalidBlockTokenException ibte) {
+ if (blockIdx > getLastRetriedIndex()) {
+ LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
+ + "for file {} for block {} from datanode {}. Will retry "
+ + "the block once.",
+ getSrc(), block, datanodes[j]);
+ setLastRetriedIndex(blockIdx);
+ done = true; // actually it's not done; but we'll retry
+ blockIdx--; // repeat at blockIdx-th block
+ setRefetchBlocks(true);
+ }
+ } catch (IOException ie) {
+ LOG.warn("src={}" + ", datanodes[{}]={}",
+ getSrc(), j, datanodes[j], ie);
+ }
+ }
+
+ return done;
+ }
+
+ /**
+ * Try one replica or datanode to compute the block checksum given a block.
+ */
+ private void tryDatanode(LocatedBlock locatedBlock,
+ DatanodeInfo datanode) throws IOException {
+
+ ExtendedBlock block = locatedBlock.getBlock();
+
+ try (IOStreamPair pair = getClient().connectToDN(datanode, getTimeout(),
+ locatedBlock.getBlockToken())) {
+
+ LOG.debug("write to {}: {}, block={}", datanode,
+ Op.BLOCK_CHECKSUM, block);
+
+ // get block MD5
+ createSender(pair).blockChecksum(block,
+ locatedBlock.getBlockToken());
+
+ final BlockOpResponseProto reply = BlockOpResponseProto.parseFrom(
+ PBHelperClient.vintPrefixed(pair.in));
+
+ String logInfo = "for block " + block + " from datanode " +
+ datanode;
+ DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);
+
+ OpBlockChecksumResponseProto checksumData =
+ reply.getChecksumResponse();
+
+ //read byte-per-checksum
+ final int bpc = checksumData.getBytesPerCrc();
+ if (blockIdx == 0) { //first block
+ setBytesPerCRC(bpc);
+ } else if (bpc != getBytesPerCRC()) {
+ throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
+ + " but bytesPerCRC=" + getBytesPerCRC());
+ }
+
+ //read crc-per-block
+ final long cpb = checksumData.getCrcPerBlock();
+ if (getLocatedBlocks().size() > 1 && blockIdx == 0) {
+ setCrcPerBlock(cpb);
+ }
+
+ //read md5
+ final MD5Hash md5 = new MD5Hash(
+ checksumData.getMd5().toByteArray());
+ md5.write(getMd5out());
+
+ // read crc-type
+ final DataChecksum.Type ct;
+ if (checksumData.hasCrcType()) {
+ ct = PBHelperClient.convert(checksumData
+ .getCrcType());
+ } else {
+ LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
+ "inferring checksum by reading first byte");
+ ct = getClient().inferChecksumTypeByReading(locatedBlock, datanode);
+ }
+
+ if (blockIdx == 0) { // first block
+ setCrcType(ct);
+ } else if (getCrcType() != DataChecksum.Type.MIXED
+ && getCrcType() != ct) {
+ // if crc types are mixed in a file
+ setCrcType(DataChecksum.Type.MIXED);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ if (blockIdx == 0) {
+ LOG.debug("set bytesPerCRC=" + getBytesPerCRC()
+ + ", crcPerBlock=" + getCrcPerBlock());
+ }
+ LOG.debug("got reply from " + datanode + ": md5=" + md5);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java
index 4157a30..4ec73e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/IOStreamPair.java
@@ -17,16 +17,19 @@
*/
package org.apache.hadoop.hdfs.protocol.datatransfer;
+import java.io.Closeable;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.IOUtils;
/**
* A little struct class to wrap an InputStream and an OutputStream.
*/
@InterfaceAudience.Private
-public class IOStreamPair {
+public class IOStreamPair implements Closeable {
public final InputStream in;
public final OutputStream out;
@@ -34,4 +37,10 @@ public class IOStreamPair {
this.in = in;
this.out = out;
}
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeStream(in);
+ IOUtils.closeStream(out);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c3ea5ce..bb847a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -233,6 +233,9 @@ Trunk (Unreleased)
HDFS-9838. Refactor the excessReplicateMap to a class. (szetszwo)
+ HDFS-9733. Refactor DFSClient#getFileChecksum and DataXceiver#blockChecksum
+ (Kai Zheng via umamahesh)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
new file mode 100644
index 0000000..9a5552d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.DataChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+
+/**
+ * Utilities for Block checksum computing, for both replicated and striped
+ * blocks.
+ */
+final class BlockChecksumHelper {
+
+ static final Logger LOG = LoggerFactory.getLogger(BlockChecksumHelper.class);
+
+ private BlockChecksumHelper() {}
+
+ /**
+ * The abstract base block checksum computer.
+ */
+ static abstract class BlockChecksumComputer {
+ private final DataNode datanode;
+ private final ExtendedBlock block;
+ // client side now can specify a range of the block for checksum
+ private final long requestLength;
+ private final LengthInputStream metadataIn;
+ private final DataInputStream checksumIn;
+ private final long visibleLength;
+ private final boolean partialBlk;
+
+ private byte[] outBytes;
+ private int bytesPerCRC = -1;
+ private DataChecksum.Type crcType = null;
+ private long crcPerBlock = -1;
+ private int checksumSize = -1;
+ private BlockMetadataHeader header;
+ private DataChecksum checksum;
+
+ BlockChecksumComputer(DataNode datanode,
+ ExtendedBlock block) throws IOException {
+ this.datanode = datanode;
+ this.block = block;
+ this.requestLength = block.getNumBytes();
+ Preconditions.checkArgument(requestLength >= 0);
+
+ this.metadataIn = datanode.data.getMetaDataInputStream(block);
+ this.visibleLength = datanode.data.getReplicaVisibleLength(block);
+ this.partialBlk = requestLength < visibleLength;
+
+ int ioFileBufferSize =
+ DFSUtilClient.getIoFileBufferSize(datanode.getConf());
+ this.checksumIn = new DataInputStream(
+ new BufferedInputStream(metadataIn, ioFileBufferSize));
+ }
+
+ protected DataNode getDatanode() {
+ return datanode;
+ }
+
+ protected ExtendedBlock getBlock() {
+ return block;
+ }
+
+ protected long getRequestLength() {
+ return requestLength;
+ }
+
+ protected LengthInputStream getMetadataIn() {
+ return metadataIn;
+ }
+
+ protected DataInputStream getChecksumIn() {
+ return checksumIn;
+ }
+
+ protected long getVisibleLength() {
+ return visibleLength;
+ }
+
+ protected boolean isPartialBlk() {
+ return partialBlk;
+ }
+
+ protected void setOutBytes(byte[] bytes) {
+ this.outBytes = bytes;
+ }
+
+ protected byte[] getOutBytes() {
+ return outBytes;
+ }
+
+ protected int getBytesPerCRC() {
+ return bytesPerCRC;
+ }
+
+ protected DataChecksum.Type getCrcType() {
+ return crcType;
+ }
+
+ protected long getCrcPerBlock() {
+ return crcPerBlock;
+ }
+
+ protected int getChecksumSize() {
+ return checksumSize;
+ }
+
+ protected BlockMetadataHeader getHeader() {
+ return header;
+ }
+
+ protected DataChecksum getChecksum() {
+ return checksum;
+ }
+
+ /**
+ * Perform the block checksum computing.
+ * @throws IOException
+ */
+ abstract void compute() throws IOException;
+
+ /**
+ * Read block metadata header.
+ * @throws IOException
+ */
+ protected void readHeader() throws IOException {
+ //read metadata file
+ header = BlockMetadataHeader.readHeader(checksumIn);
+ checksum = header.getChecksum();
+ checksumSize = checksum.getChecksumSize();
+ bytesPerCRC = checksum.getBytesPerChecksum();
+ crcPerBlock = checksumSize <= 0 ? 0 :
+ (metadataIn.getLength() -
+ BlockMetadataHeader.getHeaderSize()) / checksumSize;
+ crcType = checksum.getChecksumType();
+ }
+
+ /**
+ * Calculate partial block checksum.
+ * @return
+ * @throws IOException
+ */
+ protected byte[] crcPartialBlock() throws IOException {
+ int partialLength = (int) (requestLength % bytesPerCRC);
+ if (partialLength > 0) {
+ byte[] buf = new byte[partialLength];
+ final InputStream blockIn = datanode.data.getBlockInputStream(block,
+ requestLength - partialLength);
+ try {
+ // Get the CRC of the partialLength.
+ IOUtils.readFully(blockIn, buf, 0, partialLength);
+ } finally {
+ IOUtils.closeStream(blockIn);
+ }
+ checksum.update(buf, 0, partialLength);
+ byte[] partialCrc = new byte[checksumSize];
+ checksum.writeValue(partialCrc, 0, true);
+ return partialCrc;
+ }
+
+ return null;
+ }
+ }
+
+ /**
+ * Replicated block checksum computer.
+ */
+ static class ReplicatedBlockChecksumComputer extends BlockChecksumComputer {
+
+ ReplicatedBlockChecksumComputer(DataNode datanode,
+ ExtendedBlock block) throws IOException {
+ super(datanode, block);
+ }
+
+ @Override
+ void compute() throws IOException {
+ try {
+ readHeader();
+
+ MD5Hash md5out;
+ if (isPartialBlk() && getCrcPerBlock() > 0) {
+ md5out = checksumPartialBlock();
+ } else {
+ md5out = checksumWholeBlock();
+ }
+ setOutBytes(md5out.getDigest());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("block=" + getBlock() + ", bytesPerCRC=" + getBytesPerCRC()
+ + ", crcPerBlock=" + getCrcPerBlock() + ", md5out=" + md5out);
+ }
+ } finally {
+ IOUtils.closeStream(getChecksumIn());
+ IOUtils.closeStream(getMetadataIn());
+ }
+ }
+
+ private MD5Hash checksumWholeBlock() throws IOException {
+ MD5Hash md5out = MD5Hash.digest(getChecksumIn());
+ return md5out;
+ }
+
+ private MD5Hash checksumPartialBlock() throws IOException {
+ byte[] buffer = new byte[4*1024];
+ MessageDigest digester = MD5Hash.getDigester();
+
+ long remaining = (getRequestLength() / getBytesPerCRC())
+ * getChecksumSize();
+ for (int toDigest = 0; remaining > 0; remaining -= toDigest) {
+ toDigest = getChecksumIn().read(buffer, 0,
+ (int) Math.min(remaining, buffer.length));
+ if (toDigest < 0) {
+ break;
+ }
+ digester.update(buffer, 0, toDigest);
+ }
+
+ byte[] partialCrc = crcPartialBlock();
+ if (partialCrc != null) {
+ digester.update(partialCrc);
+ }
+
+ return new MD5Hash(digester.digest());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index ee079cb..773a64c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.SocketOutputStream;
import org.apache.hadoop.util.DataChecksum;
-import org.apache.htrace.core.Sampler;
import org.apache.htrace.core.TraceScope;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 470d3ca..3e2a25d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -3170,6 +3170,16 @@ public class DataNode extends ReconfigurableBase
return ecWorker;
}
+ IOStreamPair connectToDN(DatanodeInfo datanodeID, int timeout,
+ ExtendedBlock block,
+ Token<BlockTokenIdentifier> blockToken)
+ throws IOException {
+
+ return DFSUtilClient.connectToDN(datanodeID, timeout, conf, saslClient,
+ NetUtils.getDefaultSocketFactory(getConf()), false,
+ getDataEncryptionKeyFactoryForBlock(block), blockToken);
+ }
+
/**
* Get timeout value of each OOB type from configuration
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/307ec80a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 0041cd8..1d4a79a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -17,37 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_ACCESS_TOKEN;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_INVALID;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_UNSUPPORTED;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.SUCCESS;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.USE_RECEIPT_VERIFICATION;
-import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.DO_NOT_USE_RECEIPT_VERIFICATION;
-import static org.apache.hadoop.hdfs.server.datanode.DataNode.DN_CLIENTTRACE_FORMAT;
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InterruptedIOException;
-import java.io.OutputStream;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.nio.channels.ClosedChannelException;
-import java.security.MessageDigest;
-import java.util.Arrays;
-import java.util.concurrent.TimeUnit;
-
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
@@ -73,26 +45,52 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmR
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.BlockChecksumHelper.BlockChecksumComputer;
+import org.apache.hadoop.hdfs.server.datanode.BlockChecksumHelper.ReplicatedBlockChecksumComputer;
import org.apache.hadoop.hdfs.server.datanode.DataNode.ShortCircuitFdsUnsupportedException;
import org.apache.hadoop.hdfs.server.datanode.DataNode.ShortCircuitFdsVersionException;
import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry.NewShmInfo;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.StopWatch;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.nio.channels.ClosedChannelException;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.DO_NOT_USE_RECEIPT_VERIFICATION;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.USE_RECEIPT_VERIFICATION;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_ACCESS_TOKEN;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_INVALID;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_UNSUPPORTED;
+import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.SUCCESS;
+import static org.apache.hadoop.hdfs.server.datanode.DataNode.DN_CLIENTTRACE_FORMAT;
+import static org.apache.hadoop.util.Time.monotonicNow;
+
/**
* Thread for processing incoming/outgoing data stream.
@@ -886,90 +884,32 @@ class DataXceiver extends Receiver implements Runnable {
}
}
- private MD5Hash calcPartialBlockChecksum(ExtendedBlock block,
- long requestLength, DataChecksum checksum, DataInputStream checksumIn)
- throws IOException {
- final int bytesPerCRC = checksum.getBytesPerChecksum();
- final int csize = checksum.getChecksumSize();
- final byte[] buffer = new byte[4*1024];
- MessageDigest digester = MD5Hash.getDigester();
-
- long remaining = requestLength / bytesPerCRC * csize;
- for (int toDigest = 0; remaining > 0; remaining -= toDigest) {
- toDigest = checksumIn.read(buffer, 0,
- (int) Math.min(remaining, buffer.length));
- if (toDigest < 0) {
- break;
- }
- digester.update(buffer, 0, toDigest);
- }
-
- int partialLength = (int) (requestLength % bytesPerCRC);
- if (partialLength > 0) {
- byte[] buf = new byte[partialLength];
- final InputStream blockIn = datanode.data.getBlockInputStream(block,
- requestLength - partialLength);
- try {
- // Get the CRC of the partialLength.
- IOUtils.readFully(blockIn, buf, 0, partialLength);
- } finally {
- IOUtils.closeStream(blockIn);
- }
- checksum.update(buf, 0, partialLength);
- byte[] partialCrc = new byte[csize];
- checksum.writeValue(partialCrc, 0, true);
- digester.update(partialCrc);
- }
- return new MD5Hash(digester.digest());
- }
-
@Override
- public void blockChecksum(final ExtendedBlock block,
- final Token<BlockTokenIdentifier> blockToken) throws IOException {
+ public void blockChecksum(ExtendedBlock block,
+ Token<BlockTokenIdentifier> blockToken)
+ throws IOException {
updateCurrentThreadName("Getting checksum for block " + block);
final DataOutputStream out = new DataOutputStream(
getOutputStream());
checkAccess(out, true, block, blockToken,
Op.BLOCK_CHECKSUM, BlockTokenIdentifier.AccessMode.READ);
- // client side now can specify a range of the block for checksum
- long requestLength = block.getNumBytes();
- Preconditions.checkArgument(requestLength >= 0);
- long visibleLength = datanode.data.getReplicaVisibleLength(block);
- boolean partialBlk = requestLength < visibleLength;
-
- final LengthInputStream metadataIn = datanode.data
- .getMetaDataInputStream(block);
-
- final DataInputStream checksumIn = new DataInputStream(
- new BufferedInputStream(metadataIn, ioFileBufferSize));
+
+ BlockChecksumComputer maker =
+ new ReplicatedBlockChecksumComputer(datanode, block);
+
try {
- //read metadata file
- final BlockMetadataHeader header = BlockMetadataHeader
- .readHeader(checksumIn);
- final DataChecksum checksum = header.getChecksum();
- final int csize = checksum.getChecksumSize();
- final int bytesPerCRC = checksum.getBytesPerChecksum();
- final long crcPerBlock = csize <= 0 ? 0 :
- (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize;
-
- final MD5Hash md5 = partialBlk && crcPerBlock > 0 ?
- calcPartialBlockChecksum(block, requestLength, checksum, checksumIn)
- : MD5Hash.digest(checksumIn);
- if (LOG.isDebugEnabled()) {
- LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
- + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5);
- }
+ maker.compute();
//write reply
BlockOpResponseProto.newBuilder()
- .setStatus(SUCCESS)
- .setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()
- .setBytesPerCrc(bytesPerCRC)
- .setCrcPerBlock(crcPerBlock)
- .setMd5(ByteString.copyFrom(md5.getDigest()))
- .setCrcType(PBHelperClient.convert(checksum.getChecksumType())))
- .build()
- .writeDelimitedTo(out);
+ .setStatus(SUCCESS)
+ .setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()
+ .setBytesPerCrc(maker.getBytesPerCRC())
+ .setCrcPerBlock(maker.getCrcPerBlock())
+ .setMd5(ByteString.copyFrom(maker.getOutBytes()))
+ .setCrcType(PBHelperClient.convert(maker.getCrcType())))
+ .build()
+ .writeDelimitedTo(out);
out.flush();
} catch (IOException ioe) {
LOG.info("blockChecksum " + block + " received exception " + ioe);
@@ -977,8 +917,6 @@ class DataXceiver extends Receiver implements Runnable {
throw ioe;
} finally {
IOUtils.closeStream(out);
- IOUtils.closeStream(checksumIn);
- IOUtils.closeStream(metadataIn);
}
//update metrics
@@ -1276,7 +1214,7 @@ class DataXceiver extends Receiver implements Runnable {
/**
* Wait until the BP is registered, upto the configured amount of time.
* Throws an exception if times out, which should fail the client request.
- * @param the requested block
+ * @param block requested block
*/
void checkAndWaitForBP(final ExtendedBlock block)
throws IOException {
[38/50] [abbrv] hadoop git commit: HADOOP-12859. Disable hiding field
style checks in class setters. Contributed by Kai Zheng.
Posted by ar...@apache.org.
HADOOP-12859. Disable hiding field style checks in class setters. Contributed by Kai Zheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/480302b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/480302b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/480302b4
Branch: refs/heads/HDFS-1312
Commit: 480302b4ba932ff16b88081dac651f0a5c46c09b
Parents: 5abf051
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Mar 2 09:40:30 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Mar 2 09:40:30 2016 -0800
----------------------------------------------------------------------
hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml | 1 +
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480302b4/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 74faa48..3aaf176 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -158,6 +158,7 @@
<module name="EqualsHashCode"/>
<module name="HiddenField">
<property name="ignoreConstructorParameter" value="true"/>
+ <property name="ignoreSetter" value="true"/>
</module>
<module name="IllegalInstantiation"/>
<module name="InnerAssignment"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480302b4/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 401df94..f8e033d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -698,6 +698,8 @@ Release 2.9.0 - UNRELEASED
HADOOP-10321. TestCompositeService should cover all enumerations of
adding a service to a parent service. (Ray Chiang via kasha)
+ HADOOP-12859. Disable hiding field style checks in class setters. (Kai Zheng via wang)
+
BUG FIXES
HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing
[12/50] [abbrv] hadoop git commit: Move HADOOP-9121 from 3.0.0 to
2.7.3 in CHANGES.txt.
Posted by ar...@apache.org.
Move HADOOP-9121 from 3.0.0 to 2.7.3 in CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3ac231f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3ac231f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3ac231f
Branch: refs/heads/HDFS-1312
Commit: e3ac231f7267d8e352d2c4f39f8c5c0199e47c0c
Parents: e0b14f2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 29 11:39:43 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 29 11:40:18 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3ac231f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4739564..64ec531 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -387,9 +387,6 @@ Trunk (Unreleased)
HADOOP-9037. Bug in test-patch.sh and precommit build process (Kihwal Lee
via jlowe)
- HADOOP-9121. InodeTree.java has redundant check for vName while
- throwing exception. (Arup Malakar via suresh)
-
HADOOP-9131. Turn off TestLocalFileSystem#testListStatusWithColons on
Windows. (Chris Nauroth via suresh)
@@ -1826,6 +1823,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-8818. Use equals instead == in MD5MD5CRC32FileChecksum
and TFileDumper. (Brandon Li via suresh)
+ HADOOP-9121. InodeTree.java has redundant check for vName while
+ throwing exception. (Arup Malakar via suresh)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
[31/50] [abbrv] hadoop git commit: HADOOP-12853. Change WASB
documentation regarding page blob support. Contributed by Madhumita
Chakraborty.
Posted by ar...@apache.org.
HADOOP-12853. Change WASB documentation regarding page blob support. Contributed by Madhumita Chakraborty.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f98dff32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f98dff32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f98dff32
Branch: refs/heads/HDFS-1312
Commit: f98dff329b1f94c9f53022baf0209fc1a7aaf7c2
Parents: 5c465df
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Mar 1 13:18:42 2016 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Mar 1 13:18:42 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-tools/hadoop-azure/src/site/markdown/index.md | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98dff32/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b3413cc..401df94 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1218,6 +1218,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10865. Add a Crc32 chunked verification benchmark for both directly
and non-directly buffer cases. (szetszwo)
+ HADOOP-12853. Change WASB documentation regarding page blob support.
+ (Madhumita Chakraborty via cnauroth)
+
BUG FIXES
HADOOP-12352. Delay in checkpointing Trash can leave trash for 2 intervals
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98dff32/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index d231f22..a4a7615 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -40,8 +40,8 @@ on the additional artifacts it requires, notably the
* Present a hierarchical file system view by implementing the standard Hadoop
[`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface.
* Supports configuration of multiple Azure Blob Storage accounts.
-* Supports both page blobs (suitable for most use cases, such as MapReduce) and
- block blobs (suitable for continuous write use cases, such as an HBase
+* Supports both block blobs (suitable for most use cases, such as MapReduce) and
+ page blobs (suitable for continuous write use cases, such as an HBase
write-ahead log).
* Reference file system paths using URLs using the `wasb` scheme.
* Also reference file system paths using URLs with the `wasbs` scheme for SSL
@@ -168,9 +168,9 @@ can be written any number of times, whereas block blobs can only be appended to
work for HBase logs, so page blob support was introduced to overcome this
limitation.
-Page blobs can be used for other purposes beyond just HBase log files though.
Page blobs can be up to 1TB in size, larger than the maximum 200GB size for block
blobs.
+You should stick to block blobs for most usage, and page blobs are only tested in context of HBase write-ahead logs.
In order to have the files you create be page blobs, you must set the
configuration variable `fs.azure.page.blob.dir` to a comma-separated list of
[33/50] [abbrv] hadoop git commit: HDFS-9881.
DistributedFileSystem#getTrashRoot returns incorrect path for encryption
zones.
Posted by ar...@apache.org.
HDFS-9881. DistributedFileSystem#getTrashRoot returns incorrect path for encryption zones.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4abb2fa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4abb2fa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4abb2fa6
Branch: refs/heads/HDFS-1312
Commit: 4abb2fa687a80d2b76f2751dd31513822601b235
Parents: e76b13c
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Mar 1 16:35:39 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Mar 1 16:35:39 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 8 ++++++++
3 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4abb2fa6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index d6eb9e5..275c63d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2355,7 +2355,7 @@ public class DistributedFileSystem extends FileSystem {
EncryptionZone ez = dfs.getEZForPath(parentSrc);
if ((ez != null)) {
return this.makeQualified(
- new Path(ez.getPath(), FileSystem.TRASH_PREFIX +
+ new Path(new Path(ez.getPath(), FileSystem.TRASH_PREFIX),
dfs.ugi.getShortUserName()));
}
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4abb2fa6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 464891d..80bd4d57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2864,6 +2864,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9870. Remove unused imports from DFSUtil.
(Brahma Reddy Battula via cnauroth)
+ HDFS-9881. DistributedFileSystem#getTrashRoot returns incorrect path for
+ encryption zones. (wang)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4abb2fa6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 82a646f..6c2ce6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1445,6 +1445,14 @@ public class TestEncryptionZones {
private void verifyShellDeleteWithTrash(FsShell shell, Path path)
throws Exception{
try {
+ Path trashDir = shell.getCurrentTrashDir(path);
+ // Verify that trashDir has a path component named ".Trash"
+ Path checkTrash = trashDir;
+ while (!checkTrash.isRoot() && !checkTrash.getName().equals(".Trash")) {
+ checkTrash = checkTrash.getParent();
+ }
+ assertEquals("No .Trash component found in trash dir " + trashDir,
+ ".Trash", checkTrash.getName());
final Path trashFile =
new Path(shell.getCurrentTrashDir(path) + "/" + path);
String[] argv = new String[]{"-rm", "-r", path.toString()};
[24/50] [abbrv] hadoop git commit: HADOOP-10321. TestCompositeService
should cover all enumerations of adding a service to a parent service. (Ray
Chiang via kasha)
Posted by ar...@apache.org.
HADOOP-10321. TestCompositeService should cover all enumerations of adding a service to a parent service. (Ray Chiang via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/680f3fc0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/680f3fc0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/680f3fc0
Branch: refs/heads/HDFS-1312
Commit: 680f3fc02d0037d13c84a8b5c1a7e8729c0bcc94
Parents: d93c22e
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Feb 29 18:59:59 2016 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Feb 29 18:59:59 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/service/TestCompositeService.java | 480 +++++++++++++++++--
2 files changed, 445 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680f3fc0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8655d24..b84131b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.9.0 - UNRELEASED
HADOOP-12841. Update s3-related properties in core-default.xml. (Wei-Chiu Chuang via lei)
+ HADOOP-10321. TestCompositeService should cover all enumerations of
+ adding a service to a parent service. (Ray Chiang via kasha)
+
BUG FIXES
HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680f3fc0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
index f2ede7d..9493740 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
@@ -332,40 +332,6 @@ public class TestCompositeService {
1, testService.getServices().size());
}
- @Test(timeout = 1000)
- public void testAddInitedSiblingInInit() throws Throwable {
- CompositeService parent = new CompositeService("parent");
- BreakableService sibling = new BreakableService();
- sibling.init(new Configuration());
- parent.addService(new AddSiblingService(parent,
- sibling,
- STATE.INITED));
- parent.init(new Configuration());
- parent.start();
- parent.stop();
- assertEquals("Incorrect number of services",
- 2, parent.getServices().size());
- }
-
- @Test(timeout = 1000)
- public void testAddUninitedSiblingInInit() throws Throwable {
- CompositeService parent = new CompositeService("parent");
- BreakableService sibling = new BreakableService();
- parent.addService(new AddSiblingService(parent,
- sibling,
- STATE.INITED));
- parent.init(new Configuration());
- try {
- parent.start();
- fail("Expected an exception, got " + parent);
- } catch (ServiceStateException e) {
- //expected
- }
- parent.stop();
- assertEquals("Incorrect number of services",
- 2, parent.getServices().size());
- }
-
@Test
public void testRemoveService() {
CompositeService testService = new CompositeService("TestService") {
@@ -393,6 +359,118 @@ public class TestCompositeService {
2, testService.getServices().size());
}
+ //
+ // Tests for adding child service to parent
+ //
+
+ @Test(timeout = 1000)
+ public void testAddUninitedChildBeforeInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ AddSiblingService.addChildToService(parent, child);
+ parent.init(new Configuration());
+ assertInState(STATE.INITED, child);
+ parent.start();
+ assertInState(STATE.STARTED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddUninitedChildInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ parent.init(new Configuration());
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.NOTINITED, child);
+ try {
+ parent.start();
+ fail("Expected an exception, got " + parent);
+ } catch (ServiceStateException e) {
+ //expected
+ }
+ assertInState(STATE.NOTINITED, child);
+ parent.stop();
+ assertInState(STATE.NOTINITED, child);
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddUninitedChildInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ parent.init(new Configuration());
+ parent.start();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.NOTINITED, child);
+ parent.stop();
+ assertInState(STATE.NOTINITED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddUninitedChildInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ parent.init(new Configuration());
+ parent.start();
+ parent.stop();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.NOTINITED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedChildBeforeInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ AddSiblingService.addChildToService(parent, child);
+ parent.init(new Configuration());
+ assertInState(STATE.INITED, child);
+ parent.start();
+ assertInState(STATE.STARTED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedChildInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ parent.init(new Configuration());
+ AddSiblingService.addChildToService(parent, child);
+ parent.start();
+ assertInState(STATE.STARTED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedChildInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ parent.init(new Configuration());
+ parent.start();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.INITED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedChildInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ parent.init(new Configuration());
+ parent.start();
+ parent.stop();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.INITED, child);
+ }
+
@Test(timeout = 1000)
public void testAddStartedChildBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
@@ -407,6 +485,49 @@ public class TestCompositeService {
//expected
}
parent.stop();
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStartedChildInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ parent.init(new Configuration());
+ AddSiblingService.addChildToService(parent, child);
+ parent.start();
+ assertInState(STATE.STARTED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStartedChildInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ parent.init(new Configuration());
+ parent.start();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.STARTED, child);
+ parent.stop();
+ assertInState(STATE.STOPPED, child);
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStartedChildInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ parent.init(new Configuration());
+ parent.start();
+ parent.stop();
+ AddSiblingService.addChildToService(parent, child);
+ assertInState(STATE.STARTED, child);
}
@Test(timeout = 1000)
@@ -424,19 +545,92 @@ public class TestCompositeService {
//expected
}
parent.stop();
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
}
@Test(timeout = 1000)
- public void testAddStartedSiblingInStart() throws Throwable {
+ public void testAddStoppedChildInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ child.stop();
+ parent.init(new Configuration());
+ AddSiblingService.addChildToService(parent, child);
+ try {
+ parent.start();
+ fail("Expected an exception, got " + parent);
+ } catch (ServiceStateException e) {
+ //expected
+ }
+ assertInState(STATE.STOPPED, child);
+ parent.stop();
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedChildInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ child.stop();
+ parent.init(new Configuration());
+ parent.start();
+ AddSiblingService.addChildToService(parent, child);
+ parent.stop();
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedChildInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService child = new BreakableService();
+ child.init(new Configuration());
+ child.start();
+ child.stop();
+ parent.init(new Configuration());
+ parent.start();
+ parent.stop();
+ AddSiblingService.addChildToService(parent, child);
+ }
+
+ //
+ // Tests for adding sibling service to parent
+ //
+
+ @Test(timeout = 1000)
+ public void testAddUninitedSiblingBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
- sibling.init(new Configuration());
- sibling.start();
parent.addService(new AddSiblingService(parent,
sibling,
- STATE.STARTED));
+ STATE.NOTINITED));
parent.init(new Configuration());
+ assertInState(STATE.NOTINITED, sibling);
parent.start();
+ assertInState(STATE.NOTINITED, sibling);
+ parent.stop();
+ assertInState(STATE.NOTINITED, sibling);
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddUninitedSiblingInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.INITED));
+ parent.init(new Configuration());
+ try {
+ parent.start();
+ fail("Expected an exception, got " + parent);
+ } catch (ServiceStateException e) {
+ //expected
+ }
parent.stop();
assertEquals("Incorrect number of services",
2, parent.getServices().size());
@@ -452,12 +646,115 @@ public class TestCompositeService {
parent.init(new Configuration());
assertInState(STATE.NOTINITED, sibling);
parent.start();
+ assertInState(STATE.NOTINITED, sibling);
+ parent.stop();
+ assertInState(STATE.NOTINITED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddUninitedSiblingInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STOPPED));
+ parent.init(new Configuration());
+ assertInState(STATE.NOTINITED, sibling);
+ parent.start();
+ assertInState(STATE.NOTINITED, sibling);
+ parent.stop();
+ assertInState(STATE.NOTINITED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedSiblingBeforeInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.NOTINITED));
+ parent.init(new Configuration());
+ assertInState(STATE.INITED, sibling);
+ parent.start();
+ assertInState(STATE.INITED, sibling);
+ parent.stop();
+ assertInState(STATE.INITED, sibling);
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedSiblingInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.INITED));
+ parent.init(new Configuration());
+ assertInState(STATE.INITED, sibling);
+ parent.start();
+ assertInState(STATE.STARTED, sibling);
+ parent.stop();
+ assertInState(STATE.STOPPED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddInitedSiblingInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STARTED));
+ parent.init(new Configuration());
+ assertInState(STATE.INITED, sibling);
+ parent.start();
+ assertInState(STATE.INITED, sibling);
parent.stop();
+ assertInState(STATE.STOPPED, sibling);
assertEquals("Incorrect number of services",
2, parent.getServices().size());
}
@Test(timeout = 1000)
+ public void testAddInitedSiblingInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STOPPED));
+ parent.init(new Configuration());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStartedSiblingBeforeInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.NOTINITED));
+ parent.init(new Configuration());
+ assertInState(STATE.STARTED, sibling);
+ parent.start();
+ assertInState(STATE.STARTED, sibling);
+ parent.stop();
+ assertInState(STATE.STARTED, sibling);
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
public void testAddStartedSiblingInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -471,9 +768,29 @@ public class TestCompositeService {
parent.start();
assertInState(STATE.STARTED, sibling);
parent.stop();
+ assertInState(STATE.STOPPED, sibling);
assertEquals("Incorrect number of services",
2, parent.getServices().size());
+ }
+
+
+ @Test(timeout = 1000)
+ public void testAddStartedSiblingInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STARTED));
+ parent.init(new Configuration());
+ assertInState(STATE.STARTED, sibling);
+ parent.start();
+ assertInState(STATE.STARTED, sibling);
+ parent.stop();
assertInState(STATE.STOPPED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
}
@Test(timeout = 1000)
@@ -486,8 +803,95 @@ public class TestCompositeService {
sibling,
STATE.STOPPED));
parent.init(new Configuration());
+ assertInState(STATE.STARTED, sibling);
+ parent.start();
+ assertInState(STATE.STARTED, sibling);
+ parent.stop();
+ assertInState(STATE.STARTED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedSiblingBeforeInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ sibling.stop();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.NOTINITED));
+ parent.init(new Configuration());
+ assertInState(STATE.STOPPED, sibling);
parent.start();
+ assertInState(STATE.STOPPED, sibling);
parent.stop();
+ assertInState(STATE.STOPPED, sibling);
+ assertEquals("Incorrect number of services",
+ 1, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedSiblingInInit() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ sibling.stop();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.INITED));
+ parent.init(new Configuration());
+ assertInState(STATE.STOPPED, sibling);
+ try {
+ parent.start();
+ fail("Expected an exception, got " + parent);
+ } catch (ServiceStateException e) {
+ //expected
+ }
+ parent.stop();
+ assertInState(STATE.STOPPED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedSiblingInStart() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ sibling.stop();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STARTED));
+ parent.init(new Configuration());
+ assertInState(STATE.STOPPED, sibling);
+ parent.start();
+ assertInState(STATE.STOPPED, sibling);
+ parent.stop();
+ assertInState(STATE.STOPPED, sibling);
+ assertEquals("Incorrect number of services",
+ 2, parent.getServices().size());
+ }
+
+ @Test(timeout = 1000)
+ public void testAddStoppedSiblingInStop() throws Throwable {
+ CompositeService parent = new CompositeService("parent");
+ BreakableService sibling = new BreakableService();
+ sibling.init(new Configuration());
+ sibling.start();
+ sibling.stop();
+ parent.addService(new AddSiblingService(parent,
+ sibling,
+ STATE.STOPPED));
+ parent.init(new Configuration());
+ assertInState(STATE.STOPPED, sibling);
+ parent.start();
+ assertInState(STATE.STOPPED, sibling);
+ parent.stop();
+ assertInState(STATE.STOPPED, sibling);
assertEquals("Incorrect number of services",
2, parent.getServices().size());
}
[48/50] [abbrv] hadoop git commit: MAPREDUCE-4785. TestMRApp
occasionally fails (haibochen via rkanter)
Posted by ar...@apache.org.
MAPREDUCE-4785. TestMRApp occasionally fails (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff0ee84d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff0ee84d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff0ee84d
Branch: refs/heads/HDFS-1312
Commit: ff0ee84d77d9438f0954ae4e1497d63997bb7347
Parents: 0a9f00a
Author: Robert Kanter <rk...@cloudera.com>
Authored: Thu Mar 3 16:38:07 2016 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Thu Mar 3 16:38:07 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 2 +
.../hadoop/mapreduce/v2/app/TestMRApp.java | 104 +++++++++++++++----
2 files changed, 88 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff0ee84d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a021a0e..8939007 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -332,6 +332,8 @@ Release 2.9.0 - UNRELEASED
MAPREDUCE-6620. Jobs that did not start are shown as starting in 1969 in
the JHS web UI (haibochen via rkanter)
+ MAPREDUCE-4785. TestMRApp occasionally fails (haibochen via rkanter)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff0ee84d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
index b03d58d..eb6b932 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
@@ -25,7 +25,10 @@ import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.concurrent.TimeoutException;
+import com.google.common.base.Supplier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -205,10 +208,10 @@ public class TestMRApp {
conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.5f);
// uberization forces full slowstart (1.0), so disable that
conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
- Job job = app.submit(conf);
- app.waitForState(job, JobState.RUNNING);
- Assert.assertEquals("Num tasks not correct", 4, job.getTasks().size());
- Iterator<Task> it = job.getTasks().values().iterator();
+ final Job job1 = app.submit(conf);
+ app.waitForState(job1, JobState.RUNNING);
+ Assert.assertEquals("Num tasks not correct", 4, job1.getTasks().size());
+ Iterator<Task> it = job1.getTasks().values().iterator();
Task mapTask1 = it.next();
Task mapTask2 = it.next();
@@ -240,8 +243,20 @@ public class TestMRApp {
app.waitForState(mapTask1, TaskState.SUCCEEDED);
app.waitForState(mapTask2, TaskState.SUCCEEDED);
- TaskAttemptCompletionEvent[] events = job.getTaskAttemptCompletionEvents(0,
- 100);
+ final int checkIntervalMillis = 100;
+ final int waitForMillis = 800;
+
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job1
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 2;
+ }
+ }, checkIntervalMillis, waitForMillis);
+
+ TaskAttemptCompletionEvent[] events = job1.getTaskAttemptCompletionEvents
+ (0, 100);
Assert.assertEquals("Expecting 2 completion events for success", 2,
events.length);
@@ -253,12 +268,21 @@ public class TestMRApp {
nr.setNodeState(NodeState.UNHEALTHY);
updatedNodes.add(nr);
app.getContext().getEventHandler()
- .handle(new JobUpdatedNodesEvent(job.getID(), updatedNodes));
+ .handle(new JobUpdatedNodesEvent(job1.getID(), updatedNodes));
app.waitForState(task1Attempt, TaskAttemptState.KILLED);
app.waitForState(task2Attempt, TaskAttemptState.KILLED);
- events = job.getTaskAttemptCompletionEvents(0, 100);
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job1
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 4;
+ }
+ }, checkIntervalMillis, waitForMillis);
+
+ events = job1.getTaskAttemptCompletionEvents(0, 100);
Assert.assertEquals("Expecting 2 more completion events for killed", 4,
events.length);
@@ -281,7 +305,16 @@ public class TestMRApp {
app.waitForState(mapTask1, TaskState.SUCCEEDED);
app.waitForState(mapTask2, TaskState.RUNNING);
- events = job.getTaskAttemptCompletionEvents(0, 100);
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job1
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 5;
+ }
+ }, checkIntervalMillis, waitForMillis);
+
+ events = job1.getTaskAttemptCompletionEvents(0, 100);
Assert.assertEquals("Expecting 1 more completion events for success", 5,
events.length);
@@ -295,10 +328,11 @@ public class TestMRApp {
conf = new Configuration();
conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, true);
conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
- job = app.submit(conf);
- app.waitForState(job, JobState.RUNNING);
- Assert.assertEquals("No of tasks not correct", 4, job.getTasks().size());
- it = job.getTasks().values().iterator();
+
+ final Job job2 = app.submit(conf);
+ app.waitForState(job2, JobState.RUNNING);
+ Assert.assertEquals("No of tasks not correct", 4, job2.getTasks().size());
+ it = job2.getTasks().values().iterator();
mapTask1 = it.next();
mapTask2 = it.next();
Task reduceTask1 = it.next();
@@ -308,7 +342,16 @@ public class TestMRApp {
app.waitForState(mapTask1, TaskState.SUCCEEDED);
app.waitForState(mapTask2, TaskState.RUNNING);
- events = job.getTaskAttemptCompletionEvents(0, 100);
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job2
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 2;
+ }
+ }, checkIntervalMillis, waitForMillis);
+
+ events = job2.getTaskAttemptCompletionEvents(0, 100);
Assert.assertEquals(
"Expecting 2 completion events for killed & success of map1", 2,
events.length);
@@ -321,7 +364,16 @@ public class TestMRApp {
TaskAttemptEventType.TA_DONE));
app.waitForState(mapTask2, TaskState.SUCCEEDED);
- events = job.getTaskAttemptCompletionEvents(0, 100);
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job2
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 3;
+ }
+ }, checkIntervalMillis, waitForMillis);
+
+ events = job2.getTaskAttemptCompletionEvents(0, 100);
Assert.assertEquals("Expecting 1 more completion events for success", 3,
events.length);
@@ -350,14 +402,30 @@ public class TestMRApp {
.handle(
new TaskAttemptEvent(task4Attempt.getID(),
TaskAttemptEventType.TA_DONE));
- app.waitForState(reduceTask2, TaskState.SUCCEEDED);
+ app.waitForState(reduceTask2, TaskState.SUCCEEDED);
- events = job.getTaskAttemptCompletionEvents(0, 100);
+ waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ TaskAttemptCompletionEvent[] events = job2
+ .getTaskAttemptCompletionEvents(0, 100);
+ return events.length == 5;
+ }
+ }, checkIntervalMillis, waitForMillis);
+ events = job2.getTaskAttemptCompletionEvents(0, 100);
Assert.assertEquals("Expecting 2 more completion events for reduce success",
5, events.length);
// job succeeds
- app.waitForState(job, JobState.SUCCEEDED);
+ app.waitForState(job2, JobState.SUCCEEDED);
+ }
+
+ private static void waitFor(Supplier<Boolean> predicate, int
+ checkIntervalMillis, int checkTotalMillis) throws InterruptedException {
+ try {
+ GenericTestUtils.waitFor(predicate, checkIntervalMillis, checkTotalMillis);
+ } catch (TimeoutException ex) {
+ }
}
@Test
[17/50] [abbrv] hadoop git commit: HADOOP-12813. Migrate TestRPC and
related codes to rebase on ProtobufRpcEngine. Contributed by Kai Zheng.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
index 39e6dc5..44ed883 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
@@ -24,15 +24,25 @@ import static org.mockito.Mockito.when;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ClientId;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
+import org.apache.hadoop.ipc.TestRpcBase.TestRpcService;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -188,12 +198,19 @@ public class TestNMAuditLogger {
* A special extension of {@link TestImpl} RPC server with
* {@link TestImpl#ping()} testing the audit logs.
*/
- private class MyTestRPCServer extends TestImpl {
+ private class MyTestRPCServer extends TestRpcBase.PBServerImpl {
@Override
- public void ping() {
+ public TestProtos.EmptyResponseProto ping(
+ RpcController unused, TestProtos.EmptyRequestProto request)
+ throws ServiceException {
+ // Ensure clientId is received
+ byte[] clientId = Server.getClientId();
+ Assert.assertNotNull(clientId);
+ Assert.assertEquals(ClientId.BYTE_LENGTH, clientId.length);
// test with ip set
testSuccessLogFormat(true);
testFailureLogFormat(true);
+ return TestProtos.EmptyResponseProto.newBuilder().build();
}
}
@@ -203,9 +220,17 @@ public class TestNMAuditLogger {
@Test
public void testNMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
+ RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class);
+
+ // Create server side implementation
+ MyTestRPCServer serverImpl = new MyTestRPCServer();
+ BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto
+ .newReflectiveBlockingService(serverImpl);
+
// start the IPC server
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+ Server server = new RPC.Builder(conf)
+ .setProtocol(TestRpcBase.TestRpcService.class)
+ .setInstance(service).setBindAddress("0.0.0.0")
.setPort(0).setNumHandlers(5).setVerbose(true).build();
server.start();
@@ -213,11 +238,14 @@ public class TestNMAuditLogger {
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// Make a client connection and test the audit log
- TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
+ TestRpcService proxy = RPC.getProxy(TestRpcService.class,
TestProtocol.versionID, addr, conf);
// Start the testcase
- proxy.ping();
+ TestProtos.EmptyRequestProto pingRequest =
+ TestProtos.EmptyRequestProto.newBuilder().build();
+ proxy.ping(null, pingRequest);
server.stop();
+ RPC.stopProxy(proxy);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index 49b23d9..66af3f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -24,17 +24,27 @@ import static org.mockito.Mockito.when;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.ipc.ClientId;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.TestRpcBase.TestRpcService;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -255,12 +265,19 @@ public class TestRMAuditLogger {
* A special extension of {@link TestImpl} RPC server with
* {@link TestImpl#ping()} testing the audit logs.
*/
- private class MyTestRPCServer extends TestImpl {
+ private class MyTestRPCServer extends TestRpcBase.PBServerImpl {
@Override
- public void ping() {
+ public TestProtos.EmptyResponseProto ping(
+ RpcController unused, TestProtos.EmptyRequestProto request)
+ throws ServiceException {
+ // Ensure clientId is received
+ byte[] clientId = Server.getClientId();
+ Assert.assertNotNull(clientId);
+ Assert.assertEquals(ClientId.BYTE_LENGTH, clientId.length);
// test with ip set
testSuccessLogFormat(true);
testFailureLogFormat(true);
+ return TestProtos.EmptyResponseProto.newBuilder().build();
}
}
@@ -270,20 +287,33 @@ public class TestRMAuditLogger {
@Test
public void testRMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+
+ // Create server side implementation
+ MyTestRPCServer serverImpl = new MyTestRPCServer();
+ BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto
+ .newReflectiveBlockingService(serverImpl);
+
// start the IPC server
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+ Server server = new RPC.Builder(conf)
+ .setProtocol(TestRpcService.class)
+ .setInstance(service).setBindAddress("0.0.0.0")
.setPort(0).setNumHandlers(5).setVerbose(true).build();
+
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// Make a client connection and test the audit log
- TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
+ TestRpcService proxy = RPC.getProxy(TestRpcService.class,
+ TestProtocol.versionID, addr, conf);
// Start the testcase
- proxy.ping();
+ TestProtos.EmptyRequestProto pingRequest =
+ TestProtos.EmptyRequestProto.newBuilder().build();
+ proxy.ping(null, pingRequest);
server.stop();
+ RPC.stopProxy(proxy);
}
}
[42/50] [abbrv] hadoop git commit: HDFS-9534. Add CLI command to
clear storage policy from a path. (Contributed by Xiaobing Zhou)
Posted by ar...@apache.org.
HDFS-9534. Add CLI command to clear storage policy from a path. (Contributed by Xiaobing Zhou)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27941a18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27941a18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27941a18
Branch: refs/heads/HDFS-1312
Commit: 27941a1811831e0f2144a2f463d807755cd850b2
Parents: 700b0e4
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Mar 2 18:35:28 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Mar 2 18:35:28 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 10 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++
.../hadoop/hdfs/DistributedFileSystem.java | 25 ++
.../hadoop/hdfs/protocol/ClientProtocol.java | 13 +
.../ClientNamenodeProtocolTranslatorPB.java | 12 +
.../src/main/proto/ClientNamenodeProtocol.proto | 9 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
...tNamenodeProtocolServerSideTranslatorPB.java | 17 ++
.../hdfs/server/namenode/FSDirAttrOp.java | 74 +++--
.../hdfs/server/namenode/FSNamesystem.java | 22 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 48 +++-
.../src/site/markdown/ArchivalStorage.md | 15 ++
.../hadoop/hdfs/TestApplyingStoragePolicy.java | 268 +++++++++++++++++++
.../hdfs/tools/TestStoragePolicyCommands.java | 63 +++++
17 files changed, 585 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 8c1d57b..a96ea40 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2659,6 +2659,16 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * Unset the storage policy set for a given file or directory.
+ * @param src file or directory path.
+ * @throws IOException
+ */
+ public void unsetStoragePolicy(Path src) throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support unsetStoragePolicy");
+ }
+
+ /**
* Query the effective storage policy ID for the given file or directory.
*
* @param src file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 4ee7514..3f9aaa4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -633,6 +633,11 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public void unsetStoragePolicy(Path src) throws IOException {
+ fs.unsetStoragePolicy(src);
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
return fs.getStoragePolicy(src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 26923a8..a8795cc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -210,6 +210,8 @@ public class TestHarFileSystem {
public void setStoragePolicy(Path src, String policyName)
throws IOException;
+ public void unsetStoragePolicy(Path src) throws IOException;
+
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index da3f745..0976920 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1431,6 +1431,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
+ * Unset storage policy set for a given file/directory.
+ * @param src file/directory name
+ */
+ public void unsetStoragePolicy(String src) throws IOException {
+ checkOpen();
+ try (TraceScope ignored = newPathTraceScope("unsetStoragePolicy", src)) {
+ namenode.unsetStoragePolicy(src);
+ } catch (RemoteException e) {
+ throw e.unwrapRemoteException(AccessControlException.class,
+ FileNotFoundException.class,
+ SafeModeException.class,
+ NSQuotaExceededException.class,
+ UnresolvedPathException.class,
+ SnapshotAccessControlException.class);
+ }
+ }
+
+ /**
* @param path file/directory name
* @return Get the storage policy for specified path
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 275c63d..b601d6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -514,6 +514,31 @@ public class DistributedFileSystem extends FileSystem {
}
@Override
+ public void unsetStoragePolicy(final Path src)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ Path absF = fixRelativePart(src);
+ new FileSystemLinkResolver<Void>() {
+ @Override
+ public Void doCall(final Path p) throws IOException {
+ dfs.unsetStoragePolicy(getPathName(p));
+ return null;
+ }
+ @Override
+ public Void next(final FileSystem fs, final Path p) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).unsetStoragePolicy(p);
+ return null;
+ } else {
+ throw new UnsupportedOperationException(
+ "Cannot perform unsetStoragePolicy on a "
+ + "non-DistributedFileSystem: " + src + " -> " + p);
+ }
+ }
+ }.resolve(this, absF);
+ }
+
+ @Override
public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
statistics.incrementReadOps(1);
Path absF = fixRelativePart(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 3e82eb3..f524d7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -282,6 +282,19 @@ public interface ClientProtocol {
throws IOException;
/**
+ * Unset the storage policy set for a given file or directory.
+ * @param src Path of an existing file/directory.
+ * @throws SnapshotAccessControlException If access is denied
+ * @throws org.apache.hadoop.fs.UnresolvedLinkException if <code>src</code>
+ * contains a symlink
+ * @throws java.io.FileNotFoundException If file/dir <code>src</code> is not
+ * found
+ * @throws QuotaExceededException If changes violate the quota restriction
+ */
+ @Idempotent
+ void unsetStoragePolicy(String src) throws IOException;
+
+ /**
* Get the storage policy for a file/directory.
* @param path
* Path of an existing file/directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index d9e6472..adcc507 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -134,6 +134,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Recove
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCacheDirectiveRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotRequestProto;
@@ -1467,6 +1468,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void unsetStoragePolicy(String src) throws IOException {
+ UnsetStoragePolicyRequestProto req = UnsetStoragePolicyRequestProto
+ .newBuilder().setSrc(src).build();
+ try {
+ rpcProxy.unsetStoragePolicy(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
GetStoragePolicyRequestProto request = GetStoragePolicyRequestProto
.newBuilder().setPath(path).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 88f63d0..30732ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -113,6 +113,13 @@ message SetStoragePolicyRequestProto {
message SetStoragePolicyResponseProto { // void response
}
+message UnsetStoragePolicyRequestProto {
+ required string src = 1;
+}
+
+message UnsetStoragePolicyResponseProto {
+}
+
message GetStoragePolicyRequestProto {
required string path = 1;
}
@@ -745,6 +752,8 @@ service ClientNamenodeProtocol {
returns(SetReplicationResponseProto);
rpc setStoragePolicy(SetStoragePolicyRequestProto)
returns(SetStoragePolicyResponseProto);
+ rpc unsetStoragePolicy(UnsetStoragePolicyRequestProto)
+ returns(UnsetStoragePolicyResponseProto);
rpc getStoragePolicy(GetStoragePolicyRequestProto)
returns(GetStoragePolicyResponseProto);
rpc getStoragePolicies(GetStoragePoliciesRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 728342a..d922cd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1047,6 +1047,9 @@ Release 2.9.0 - UNRELEASED
HDFS-7964. Add support for async edit logging. (Daryn Sharp)
+ HDFS-9534. Add CLI command to clear storage policy from a path.
+ (Xiaobing Zhou via Arpit Agarwal)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 13c0b14..da7c524 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -157,6 +157,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Remove
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCacheDirectiveResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RemoveCachePoolResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UnsetStoragePolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2ResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
@@ -262,6 +264,9 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetSnapshottableDirListingResponseProto.newBuilder().build();
static final SetStoragePolicyResponseProto VOID_SET_STORAGE_POLICY_RESPONSE =
SetStoragePolicyResponseProto.newBuilder().build();
+ static final UnsetStoragePolicyResponseProto
+ VOID_UNSET_STORAGE_POLICY_RESPONSE =
+ UnsetStoragePolicyResponseProto.newBuilder().build();
private static final CreateResponseProto VOID_CREATE_RESPONSE =
CreateResponseProto.newBuilder().build();
@@ -1486,6 +1491,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public UnsetStoragePolicyResponseProto unsetStoragePolicy(
+ RpcController controller, UnsetStoragePolicyRequestProto request)
+ throws ServiceException {
+ try {
+ server.unsetStoragePolicy(request.getSrc());
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return VOID_UNSET_STORAGE_POLICY_RESPONSE;
+ }
+
+ @Override
public GetStoragePolicyResponseProto getStoragePolicy(
RpcController controller, GetStoragePolicyRequestProto request)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 2cba2cb..06942f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
+import com.google.common.collect.Lists;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
@@ -158,13 +160,31 @@ public class FSDirAttrOp {
return isFile;
}
- static HdfsFileStatus setStoragePolicy(
- FSDirectory fsd, BlockManager bm, String src, final String policyName)
+ static HdfsFileStatus unsetStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src) throws IOException {
+ return setStoragePolicy(fsd, bm, src,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, "unset");
+ }
+
+ static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src, final String policyName) throws IOException {
+ // get the corresponding policy and make sure the policy name is valid
+ BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
+ if (policy == null) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot find a block policy with the name " + policyName);
+ }
+
+ return setStoragePolicy(fsd, bm, src, policy.getId(), "set");
+ }
+
+ static HdfsFileStatus setStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src, final byte policyId, final String operation)
throws IOException {
if (!fsd.isStoragePolicyEnabled()) {
- throw new IOException(
- "Failed to set storage policy since "
- + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
+ throw new IOException(String.format(
+ "Failed to %s storage policy since %s is set to false.", operation,
+ DFS_STORAGE_POLICY_ENABLED_KEY));
}
FSPermissionChecker pc = fsd.getPermissionChecker();
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
@@ -178,14 +198,8 @@ public class FSDirAttrOp {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
}
- // get the corresponding policy and make sure the policy name is valid
- BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
- if (policy == null) {
- throw new HadoopIllegalArgumentException(
- "Cannot find a block policy with the name " + policyName);
- }
- unprotectedSetStoragePolicy(fsd, bm, iip, policy.getId());
- fsd.getEditLog().logSetStoragePolicy(src, policy.getId());
+ unprotectedSetStoragePolicy(fsd, bm, iip, policyId);
+ fsd.getEditLog().logSetStoragePolicy(src, policyId);
} finally {
fsd.writeUnlock();
}
@@ -446,8 +460,8 @@ public class FSDirAttrOp {
return file.getBlocks();
}
- static void unprotectedSetStoragePolicy(
- FSDirectory fsd, BlockManager bm, INodesInPath iip, byte policyId)
+ static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm,
+ INodesInPath iip, final byte policyId)
throws IOException {
assert fsd.hasWriteLock();
final INode inode = iip.getLastINode();
@@ -457,10 +471,12 @@ public class FSDirAttrOp {
}
final int snapshotId = iip.getLatestSnapshotId();
if (inode.isFile()) {
- BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
- if (newPolicy.isCopyOnCreateFile()) {
- throw new HadoopIllegalArgumentException(
- "Policy " + newPolicy + " cannot be set after file creation.");
+ if (policyId != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
+ if (newPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException("Policy " + newPolicy
+ + " cannot be set after file creation.");
+ }
}
BlockStoragePolicy currentPolicy =
@@ -473,7 +489,8 @@ public class FSDirAttrOp {
}
inode.asFile().setStoragePolicyID(policyId, snapshotId);
} else if (inode.isDirectory()) {
- setDirStoragePolicy(fsd, inode.asDirectory(), policyId, snapshotId);
+ setDirStoragePolicy(fsd, inode.asDirectory(), policyId,
+ snapshotId);
} else {
throw new FileNotFoundException(iip.getPath()
+ " is not a file or directory");
@@ -485,11 +502,18 @@ public class FSDirAttrOp {
int latestSnapshotId) throws IOException {
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
- List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
- Arrays.asList(xAttr),
- EnumSet.of(
- XAttrSetFlag.CREATE,
- XAttrSetFlag.REPLACE));
+ List<XAttr> newXAttrs = null;
+ if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ List<XAttr> toRemove = Lists.newArrayList();
+ toRemove.add(xAttr);
+ List<XAttr> removed = Lists.newArrayList();
+ newXAttrs = FSDirXAttrOp.filterINodeXAttrs(existingXAttrs, toRemove,
+ removed);
+ } else {
+ newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
+ Arrays.asList(xAttr),
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ }
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 31f2b93..907a0ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1961,6 +1961,28 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * unset storage policy set for a given file or a directory.
+ *
+ * @param src file/directory path
+ */
+ void unsetStoragePolicy(String src) throws IOException {
+ HdfsFileStatus auditStat;
+ checkOperation(OperationCategory.WRITE);
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot unset storage policy for " + src);
+ auditStat = FSDirAttrOp.unsetStoragePolicy(dir, blockManager, src);
+ } catch (AccessControlException e) {
+ logAuditEvent(false, "unsetStoragePolicy", src);
+ throw e;
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ logAuditEvent(true, "unsetStoragePolicy", src, null, auditStat);
+ }
+ /**
* Get the storage policy for a file or a directory.
*
* @param src
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index c1646c5..275e210 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -771,6 +771,13 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
@Override
+ public void unsetStoragePolicy(String src)
+ throws IOException {
+ checkNNStartup();
+ namesystem.unsetStoragePolicy(src);
+ }
+
+ @Override
public void setStoragePolicy(String src, String policyName)
throws IOException {
checkNNStartup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 98c8a6b..24079b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -224,9 +224,55 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
}
+ /* Command to unset the storage policy set for a file/directory */
+ private static class UnsetStoragePolicyCommand
+ implements AdminHelper.Command {
+
+ @Override
+ public String getName() {
+ return "-unsetStoragePolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the file/directory "
+ + "from which the storage policy will be unset.");
+ return getShortUsage() + "\n"
+ + "Unset the storage policy set for a file/directory.\n\n"
+ + listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path from which "
+ + "the storage policy will be unsetd.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ try {
+ dfs.unsetStoragePolicy(new Path(path));
+ System.out.println("Unset storage policy from " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
private static final AdminHelper.Command[] COMMANDS = {
new ListStoragePoliciesCommand(),
new SetStoragePolicyCommand(),
- new GetStoragePolicyCommand()
+ new GetStoragePolicyCommand(),
+ new UnsetStoragePolicyCommand()
};
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
index cbfbaa1..803cc91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
@@ -26,6 +26,7 @@ Archival Storage, SSD & Memory
* [Storage Policy Commands](#Storage_Policy_Commands)
* [List Storage Policies](#List_Storage_Policies)
* [Set Storage Policy](#Set_Storage_Policy)
+ * [Unset Storage Policy](#Unset_Storage_Policy)
* [Get Storage Policy](#Get_Storage_Policy)
Introduction
@@ -150,6 +151,20 @@ Set a storage policy to a file or a directory.
| `-path <path>` | The path referring to either a directory or a file. |
| `-policy <policy>` | The name of the storage policy. |
+### Unset Storage Policy
+
+Unset a storage policy to a file or a directory. After the unset command the storage policy of the nearest ancestor will apply, and if there is no policy on any ancestor then the default storage policy will apply.
+
+* Command:
+
+ hdfs storagepolicies -unsetStoragePolicy -path <path>
+
+* Arguments:
+
+| | |
+|:---- |:---- |
+| `-path <path>` | The path referring to either a directory or a file. |
+
### Get Storage Policy
Get the storage policy of a file or a directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
new file mode 100644
index 0000000..200fab6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestApplyingStoragePolicy.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestApplyingStoragePolicy {
+ private static final short REPL = 1;
+ private static final int SIZE = 128;
+
+ private static Configuration conf;
+ private static MiniDFSCluster cluster;
+ private static DistributedFileSystem fs;
+
+ @Before
+ public void clusterSetUp() throws IOException {
+ conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void clusterShutdown() throws IOException{
+ if(fs != null) {
+ fs.close();
+ fs = null;
+ }
+ if(cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testStoragePolicyByDefault() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path(bar, "/fooz");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: storage policy is HOT by default or inherited from nearest
+ * ancestor, if not explicitly specified for newly created dir/file.
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testSetAndUnsetStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path(bar, "/fooz");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ fs.setStoragePolicy(wow, hot.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: unset storage policy
+ */
+ fs.unsetStoragePolicy(foo);
+ fs.unsetStoragePolicy(bar);
+ fs.unsetStoragePolicy(wow);
+ try {
+ fs.unsetStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after unset
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testNestedStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ final Path fooz = new Path("/foos");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ fs.setStoragePolicy(wow, hot.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: unset storage policy in the case of being nested
+ */
+ // unset wow
+ fs.unsetStoragePolicy(wow);
+ // inherit storage policy from wow's nearest ancestor
+ assertEquals(fs.getStoragePolicy(wow), cold);
+ // unset bar
+ fs.unsetStoragePolicy(bar);
+ // inherit storage policy from bar's nearest ancestor
+ assertEquals(fs.getStoragePolicy(bar), warm);
+ // unset foo
+ fs.unsetStoragePolicy(foo);
+ // default storage policy is applied, since no more available ancestors
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ // unset fooz
+ try {
+ fs.unsetStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: default storage policy is applied, since no explicit policies from
+ * ancestors are available
+ */
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ assertEquals(fs.getStoragePolicy(wow), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+
+ @Test
+ public void testSetAndGetStoragePolicy() throws IOException {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path fooz = new Path("/fooz");
+ DFSTestUtil.createFile(fs, bar, SIZE, REPL, 0);
+
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+
+ assertEquals(fs.getStoragePolicy(foo), hot);
+ assertEquals(fs.getStoragePolicy(bar), hot);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: set storage policy
+ */
+ fs.setStoragePolicy(foo, warm.getName());
+ fs.setStoragePolicy(bar, cold.getName());
+ try {
+ fs.setStoragePolicy(fooz, warm.getName());
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+
+ /*
+ * test: get storage policy after set
+ */
+ assertEquals(fs.getStoragePolicy(foo), warm);
+ assertEquals(fs.getStoragePolicy(bar), cold);
+ try {
+ fs.getStoragePolicy(fooz);
+ } catch (Exception e) {
+ assertTrue(e instanceof FileNotFoundException);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27941a18/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
index ec0bb66..63f8484 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
@@ -62,6 +62,69 @@ public class TestStoragePolicyCommands {
}
}
+
+ @Test
+ public void testSetAndUnsetStoragePolicy() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final Path wow = new Path(bar, "wow");
+ DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
+
+ /*
+ * test: set storage policy
+ */
+ final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0,
+ "Set storage policy WARM on " + foo.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD",
+ 0, "Set storage policy COLD on " + bar.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo/bar/wow -policy HOT",
+ 0, "Set storage policy HOT on " + wow.toString());
+ DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /fooz -policy WARM",
+ 2, "File/Directory does not exist: /fooz");
+
+ /*
+ * test: get storage policy after set
+ */
+ final BlockStoragePolicySuite suite = BlockStoragePolicySuite
+ .createDefaultSuite();
+ final BlockStoragePolicy warm = suite.getPolicy("WARM");
+ final BlockStoragePolicy cold = suite.getPolicy("COLD");
+ final BlockStoragePolicy hot = suite.getPolicy("HOT");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+ "The storage policy of " + foo.toString() + ":\n" + warm);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+ "The storage policy of " + bar.toString() + ":\n" + cold);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar/wow", 0,
+ "The storage policy of " + wow.toString() + ":\n" + hot);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+
+ /*
+ * test: unset storage policy
+ */
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo", 0,
+ "Unset storage policy from " + foo.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo/bar", 0,
+ "Unset storage policy from " + bar.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /foo/bar/wow", 0,
+ "Unset storage policy from " + wow.toString());
+ DFSTestUtil.toolRun(admin, "-unsetStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+
+ /*
+ * test: get storage policy after unset
+ */
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+ "The storage policy of " + foo.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+ "The storage policy of " + bar.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar/wow", 0,
+ "The storage policy of " + wow.toString() + " is unspecified");
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /fooz", 2,
+ "File/Directory does not exist: /fooz");
+ }
+
@Test
public void testSetAndGetStoragePolicy() throws Exception {
final Path foo = new Path("/foo");
[34/50] [abbrv] hadoop git commit: HDFS-9876.
shouldProcessOverReplicated should not count number of pending replicas.
Contributed by Jing Zhao.
Posted by ar...@apache.org.
HDFS-9876. shouldProcessOverReplicated should not count number of pending replicas. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2ba7da4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2ba7da4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2ba7da4
Branch: refs/heads/HDFS-1312
Commit: f2ba7da4f0df6cf0fc245093aeb4500158e6ee0b
Parents: 4abb2fa
Author: Jing Zhao <ji...@apache.org>
Authored: Tue Mar 1 18:41:57 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Tue Mar 1 18:41:57 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/blockmanagement/BlockManager.java | 38 ++++++++++----------
2 files changed, 23 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2ba7da4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 80bd4d57..421ee68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -455,6 +455,9 @@ Trunk (Unreleased)
HDFS-9867. Missing block exception should carry locatedBlocks information.
(Mingliang Liu via jing9)
+ HDFS-9876. shouldProcessOverReplicated should not count number of pending
+ replicas. (jing9)
+
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2ba7da4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 77eea0a..5175c13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2893,7 +2893,7 @@ public class BlockManager implements BlockStatsMXBean {
} else {
updateNeededReplications(storedBlock, curReplicaDelta, 0);
}
- if (shouldProcessOverReplicated(num, pendingNum, fileReplication)) {
+ if (shouldProcessOverReplicated(num, fileReplication)) {
processOverReplicatedBlock(storedBlock, fileReplication, node, delNodeHint);
}
// If the file replication has reached desired value
@@ -2912,8 +2912,8 @@ public class BlockManager implements BlockStatsMXBean {
}
private boolean shouldProcessOverReplicated(NumberReplicas num,
- int pendingNum, int expectedNum) {
- int numCurrent = num.liveReplicas() + pendingNum;
+ int expectedNum) {
+ final int numCurrent = num.liveReplicas();
return numCurrent > expectedNum ||
(numCurrent == expectedNum && num.redundantInternalBlocks() > 0);
}
@@ -3131,7 +3131,7 @@ public class BlockManager implements BlockStatsMXBean {
}
}
- if (shouldProcessOverReplicated(num, 0, expectedReplication)) {
+ if (shouldProcessOverReplicated(num, expectedReplication)) {
if (num.replicasOnStaleNodes() > 0) {
// If any of the replicas of this block are on nodes that are
// considered "stale", then these replicas may in fact have
@@ -3268,7 +3268,6 @@ public class BlockManager implements BlockStatsMXBean {
assert storedBlock instanceof BlockInfoStriped;
BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
short groupSize = sblk.getTotalBlockNum();
- BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
// find all duplicated indices
BitSet found = new BitSet(groupSize); //indices found
@@ -3283,14 +3282,6 @@ public class BlockManager implements BlockStatsMXBean {
found.set(index);
storage2index.put(storage, index);
}
- // the number of target left replicas equals to the of number of the found
- // indices.
- int numOfTarget = found.cardinality();
-
- final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
- bc.getStoragePolicyID());
- final List<StorageType> excessTypes = storagePolicy.chooseExcess(
- (short)numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
// use delHint only if delHint is duplicated
final DatanodeStorageInfo delStorageHint =
@@ -3302,6 +3293,19 @@ public class BlockManager implements BlockStatsMXBean {
}
}
+ // cardinality of found indicates the expected number of internal blocks
+ final int numOfTarget = found.cardinality();
+ final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
+ bc.getStoragePolicyID());
+ final List<StorageType> excessTypes = storagePolicy.chooseExcess(
+ (short) numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
+ if (excessTypes.isEmpty()) {
+ LOG.warn("excess types chosen for block {} among storages {} is empty",
+ storedBlock, nonExcess);
+ return;
+ }
+
+ BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
// for each duplicated index, delete some replicas until only one left
for (int targetIndex = duplicated.nextSetBit(0); targetIndex >= 0;
targetIndex = duplicated.nextSetBit(targetIndex + 1)) {
@@ -3312,9 +3316,7 @@ public class BlockManager implements BlockStatsMXBean {
candidates.add(storage);
}
}
- Block internalBlock = new Block(storedBlock);
- internalBlock.setBlockId(storedBlock.getBlockId() + targetIndex);
- while (candidates.size() > 1) {
+ if (candidates.size() > 1) {
List<DatanodeStorageInfo> replicasToDelete = placementPolicy
.chooseReplicasToDelete(nonExcess, candidates, (short) 1,
excessTypes, null, null);
@@ -3749,7 +3751,7 @@ public class BlockManager implements BlockStatsMXBean {
final BlockInfo block = it.next();
int expectedReplication = this.getReplication(block);
NumberReplicas num = countNodes(block);
- if (shouldProcessOverReplicated(num, 0, expectedReplication)) {
+ if (shouldProcessOverReplicated(num, expectedReplication)) {
// over-replicated block
processOverReplicatedBlock(block, (short) expectedReplication, null,
null);
@@ -3886,7 +3888,7 @@ public class BlockManager implements BlockStatsMXBean {
neededReplications.add(block, n.liveReplicas() + pending,
n.readOnlyReplicas(),
n.decommissionedAndDecommissioning(), expected);
- } else if (shouldProcessOverReplicated(n, 0, expected)) {
+ } else if (shouldProcessOverReplicated(n, expected)) {
processOverReplicatedBlock(block, expected, null, null);
}
}
[32/50] [abbrv] hadoop git commit: HDFS-9880.
TestDatanodeRegistration fails occasionally. Contributed by Kihwal Lee.
Posted by ar...@apache.org.
HDFS-9880. TestDatanodeRegistration fails occasionally. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e76b13c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e76b13c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e76b13c4
Branch: refs/heads/HDFS-1312
Commit: e76b13c415459e4062c4c9660a16759a11ffb34a
Parents: f98dff3
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Mar 1 17:00:18 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Mar 1 17:00:18 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java | 6 +++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e76b13c4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 39650e8..464891d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2963,6 +2963,8 @@ Release 2.7.3 - UNRELEASED
HDFS-9864. Correct reference for RENEWDELEGATIONTOKEN and
CANCELDELEGATIONTOKEN in webhdfs doc. (Brahma Reddy Battula via aajisaka)
+ HDFS-9880. TestDatanodeRegistration fails occasionally (kihwal)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e76b13c4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
index 5870321..b6ae281 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
@@ -324,7 +324,7 @@ public class TestDatanodeRegistration {
// should not occur other than dead/unregistered node which will trigger a
// re-registration. If a non-IPC exception does occur, the safety net is
// a forced re-registration on the next heartbeat.
- @Test(timeout=10000)
+ @Test
public void testForcedRegistration() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HANDLER_COUNT_KEY, 4);
@@ -439,10 +439,10 @@ public class TestDatanodeRegistration {
public Boolean get() {
return lastCount != storage.getBlockReportCount();
}
- }, 10, 100);
+ }, 10, 2000);
} catch (TimeoutException te) {
return false;
}
return true;
}
-}
\ No newline at end of file
+}
[19/50] [abbrv] hadoop git commit: HDFS-7964. Add support for async
edit logging. Contributed by Daryn Sharp.
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
index 0265a4d..87e2523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
@@ -27,6 +27,8 @@ import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -50,13 +52,38 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Sets;
/**
* This tests data recovery mode for the NameNode.
*/
+
+@RunWith(Parameterized.class)
public class TestNameNodeRecovery {
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestNameNodeRecovery(Boolean async) {
+ useAsyncEditLog = async;
+ }
+
+ private static Configuration getConf() {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
+ return conf;
+ }
+
private static final Log LOG = LogFactory.getLog(TestNameNodeRecovery.class);
private static final StartupOption recoverStartOpt = StartupOption.RECOVER;
private static final File TEST_DIR = PathUtils.getTestDir(TestNameNodeRecovery.class);
@@ -73,7 +100,7 @@ public class TestNameNodeRecovery {
EditLogFileOutputStream elfos = null;
EditLogFileInputStream elfis = null;
try {
- elfos = new EditLogFileOutputStream(new Configuration(), TEST_LOG_NAME, 0);
+ elfos = new EditLogFileOutputStream(getConf(), TEST_LOG_NAME, 0);
elfos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
elts.addTransactionsToLog(elfos, cache);
@@ -525,7 +552,7 @@ public class TestNameNodeRecovery {
final boolean needRecovery = corruptor.needRecovery(finalize);
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
setupRecoveryTestConf(conf);
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index 5a104ad..30db429 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -32,6 +34,7 @@ import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -40,11 +43,31 @@ import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Supplier;
+@RunWith(Parameterized.class)
public class TestEditLogTailer {
-
+ static {
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestEditLogTailer(Boolean async) {
+ useAsyncEditLog = async;
+ }
+
private static final String DIR_PREFIX = "/dir";
private static final int DIRS_TO_MAKE = 20;
static final long SLEEP_TIME = 1000;
@@ -52,13 +75,21 @@ public class TestEditLogTailer {
static {
GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
GenericTestUtils.setLogLevel(EditLogTailer.LOG, Level.ALL);
}
-
+
+ private static Configuration getConf() {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
+ return conf;
+ }
+
@Test
public void testTailer() throws IOException, InterruptedException,
ServiceFailedException {
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
@@ -122,7 +153,7 @@ public class TestEditLogTailer {
private static void testStandbyTriggersLogRolls(int activeIndex)
throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
// Roll every 1s
conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
[03/50] [abbrv] hadoop git commit: HADOOP-12849.
TestSymlinkLocalFSFileSystem fails intermittently. Contributed by Mingliang
Liu.
Posted by ar...@apache.org.
HADOOP-12849. TestSymlinkLocalFSFileSystem fails intermittently. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/798babf6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/798babf6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/798babf6
Branch: refs/heads/HDFS-1312
Commit: 798babf661311264fa895c89a33d0f2da927da33
Parents: 0f72da7
Author: Chris Nauroth <cn...@apache.org>
Authored: Fri Feb 26 21:59:08 2016 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Fri Feb 26 21:59:08 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java | 6 ++++--
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/798babf6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 106dc64..79cf4d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1752,6 +1752,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12716. KerberosAuthenticator#doSpnegoSequence use incorrect class to
determine isKeyTab in JDK8. (Xiaoyu Yao via cnauroth)
+ HADOOP-12849. TestSymlinkLocalFSFileSystem fails intermittently.
+ (Mingliang Liu via cnauroth)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/798babf6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index 007c37a..a5808e6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -1373,7 +1373,8 @@ public abstract class SymlinkBaseTest {
long at = wrapper.getFileLinkStatus(link).getAccessTime();
// the local file system may not support millisecond timestamps
wrapper.setTimes(link, 2000L, 3000L);
- assertEquals(at, wrapper.getFileLinkStatus(link).getAccessTime());
+ assertTrue("The atime of symlink should not be lesser after setTimes()",
+ wrapper.getFileLinkStatus(link).getAccessTime() >= at);
assertEquals(2000, wrapper.getFileStatus(file).getModificationTime());
assertEquals(3000, wrapper.getFileStatus(file).getAccessTime());
}
@@ -1407,6 +1408,7 @@ public abstract class SymlinkBaseTest {
} catch (IOException e) {
// Expected
}
- assertEquals(at, wrapper.getFileLinkStatus(link).getAccessTime());
+ assertTrue("The atime of symlink should not be lesser after setTimes()",
+ wrapper.getFileLinkStatus(link).getAccessTime() >= at);
}
}
[02/50] [abbrv] hadoop git commit: MAPREDUCE-6622. Add capability to
set JHS job cache to a task-based limit (rchiang via rkanter)
Posted by ar...@apache.org.
MAPREDUCE-6622. Add capability to set JHS job cache to a task-based limit (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f72da7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f72da7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f72da7e
Branch: refs/heads/HDFS-1312
Commit: 0f72da7e281376f4fcbfbf3fb33f5d7fedcdb1aa
Parents: d1d4e16
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Feb 26 17:57:17 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Feb 26 17:57:17 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 8 +-
.../src/main/resources/mapred-default.xml | 27 +-
.../mapreduce/v2/hs/CachedHistoryStorage.java | 146 ++++++++---
.../hadoop/mapreduce/v2/hs/TestJobHistory.java | 255 ++++++++++++++++++-
5 files changed, 381 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f72da7e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index b8eabb2..a021a0e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -322,6 +322,9 @@ Release 2.9.0 - UNRELEASED
MAPREDUCE-6640. mapred job -history command should be able to take
Job ID (rkanter)
+ MAPREDUCE-6622. Add capability to set JHS job cache to a
+ task-based limit (rchiang via rkanter)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f72da7e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 2277ef6..1f2088a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -98,7 +98,7 @@ public class JHAdminConfig {
public static final String MR_HISTORY_JOBLIST_CACHE_SIZE =
MR_HISTORY_PREFIX + "joblist.cache.size";
public static final int DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE = 20000;
-
+
/** The location of the Kerberos keytab file.*/
public static final String MR_HISTORY_KEYTAB = MR_HISTORY_PREFIX + "keytab";
@@ -106,7 +106,11 @@ public class JHAdminConfig {
public static final String MR_HISTORY_LOADED_JOB_CACHE_SIZE =
MR_HISTORY_PREFIX + "loadedjobs.cache.size";
public static final int DEFAULT_MR_HISTORY_LOADED_JOB_CACHE_SIZE = 5;
-
+
+ /** Size of the loaded job cache (in tasks).*/
+ public static final String MR_HISTORY_LOADED_TASKS_CACHE_SIZE =
+ MR_HISTORY_PREFIX + "loadedtasks.cache.size";
+
/**
* The maximum age of a job history file before it is deleted from the history
* server.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f72da7e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index d87085f..dc5c3dd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1656,7 +1656,32 @@
<property>
<name>mapreduce.jobhistory.loadedjobs.cache.size</name>
<value>5</value>
- <description>Size of the loaded job cache</description>
+ <description>Size of the loaded job cache. This property is ignored if
+ the property mapreduce.jobhistory.loadedtasks.cache.size is set to a
+ positive value.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.jobhistory.loadedtasks.cache.size</name>
+ <value></value>
+ <description>Change the job history cache limit to be set in terms
+ of total task count. If the total number of tasks loaded exceeds
+ this value, then the job cache will be shrunk down until it is
+ under this limit (minimum 1 job in cache). If this value is empty
+ or nonpositive then the cache reverts to using the property
+ mapreduce.jobhistory.loadedjobs.cache.size as a job cache size.
+
+ Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size
+ property:
+ 1) For every 100k of cache size, set the heap size of the Job History
+ Server to 1.2GB. For example,
+ mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB.
+ 2) Make sure that the cache size is larger than the number of tasks
+ required for the largest job run on the cluster. It might be a good
+ idea to set the value slightly higher (say, 20%) in order to allow
+ for job size growth.
+ </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f72da7e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
index 8c9abc3..c59d17f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
@@ -20,12 +20,16 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.Weigher;
+import com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -49,9 +53,10 @@ public class CachedHistoryStorage extends AbstractService implements
HistoryStorage {
private static final Log LOG = LogFactory.getLog(CachedHistoryStorage.class);
- private Map<JobId, Job> loadedJobCache = null;
- // The number of loaded jobs.
+ private LoadingCache<JobId, Job> loadedJobCache = null;
private int loadedJobCacheSize;
+ private int loadedTasksCacheSize;
+ private boolean useLoadedTasksCache;
private HistoryFileManager hsManager;
@@ -70,17 +75,70 @@ public class CachedHistoryStorage extends AbstractService implements
@SuppressWarnings("serial")
private void createLoadedJobCache(Configuration conf) {
+ // Set property for old "loaded jobs" cache
loadedJobCacheSize = conf.getInt(
JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_LOADED_JOB_CACHE_SIZE);
- loadedJobCache = Collections.synchronizedMap(new LinkedHashMap<JobId, Job>(
- loadedJobCacheSize + 1, 0.75f, true) {
+ // Check property for new "loaded tasks" cache perform sanity checking
+ useLoadedTasksCache = false;
+ try {
+ String taskSizeString = conf
+ .get(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE);
+ if (taskSizeString != null) {
+ loadedTasksCacheSize = Math.max(Integer.parseInt(taskSizeString), 1);
+ useLoadedTasksCache = true;
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.error("The property " +
+ JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE +
+ " is not an integer value. Please set it to a positive" +
+ " integer value.");
+ }
+
+ CacheLoader<JobId, Job> loader;
+ loader = new CacheLoader<JobId, Job>() {
@Override
- public boolean removeEldestEntry(final Map.Entry<JobId, Job> eldest) {
- return super.size() > loadedJobCacheSize;
+ public Job load(JobId key) throws Exception {
+ return loadJob(key);
}
- });
+ };
+
+ if (!useLoadedTasksCache) {
+ loadedJobCache = CacheBuilder.newBuilder()
+ .maximumSize(loadedJobCacheSize)
+ .initialCapacity(loadedJobCacheSize)
+ .concurrencyLevel(1)
+ .build(loader);
+ } else {
+ Weigher<JobId, Job> weightByTasks;
+ weightByTasks = new Weigher<JobId, Job>() {
+ /**
+ * Method for calculating Job weight by total task count. If
+ * the total task count is greater than the size of the tasks
+ * cache, then cap it at the cache size. This allows the cache
+ * to always hold one large job.
+ * @param key JobId object
+ * @param value Job object
+ * @return Weight of the job as calculated by total task count
+ */
+ @Override
+ public int weigh(JobId key, Job value) {
+ int taskCount = Math.min(loadedTasksCacheSize,
+ value.getTotalMaps() + value.getTotalReduces());
+ return taskCount;
+ }
+ };
+ // Keep concurrencyLevel at 1. Otherwise, two problems:
+ // 1) The largest job that can be initially loaded is
+ // cache size / 4.
+ // 2) Unit tests are not deterministic.
+ loadedJobCache = CacheBuilder.newBuilder()
+ .maximumWeight(loadedTasksCacheSize)
+ .weigher(weightByTasks)
+ .concurrencyLevel(1)
+ .build(loader);
+ }
}
public void refreshLoadedJobCache() {
@@ -100,52 +158,48 @@ public class CachedHistoryStorage extends AbstractService implements
public CachedHistoryStorage() {
super(CachedHistoryStorage.class.getName());
}
+
+ private static class HSFileRuntimeException extends RuntimeException {
+ public HSFileRuntimeException(String message) {
+ super(message);
+ }
+ }
- private Job loadJob(HistoryFileInfo fileInfo) {
- try {
- Job job = fileInfo.loadJob();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding " + job.getID() + " to loaded job cache");
- }
- // We can clobber results here, but that should be OK, because it only
- // means that we may have two identical copies of the same job floating
- // around for a while.
- loadedJobCache.put(job.getID(), job);
- return job;
- } catch (IOException e) {
- throw new YarnRuntimeException(
- "Could not find/load job: " + fileInfo.getJobId(), e);
+ private Job loadJob(JobId jobId) throws RuntimeException, IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking for Job " + jobId);
+ }
+ HistoryFileInfo fileInfo;
+
+ fileInfo = hsManager.getFileInfo(jobId);
+ if (fileInfo == null) {
+ throw new HSFileRuntimeException("Unable to find job " + jobId);
+ } else if (fileInfo.isDeleted()) {
+ throw new HSFileRuntimeException("Cannot load deleted job " + jobId);
+ } else {
+ return fileInfo.loadJob();
}
}
@VisibleForTesting
- Map<JobId, Job> getLoadedJobCache() {
+ Cache<JobId, Job> getLoadedJobCache() {
return loadedJobCache;
}
@Override
public Job getFullJob(JobId jobId) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Looking for Job " + jobId);
- }
+ Job retVal = null;
try {
- HistoryFileInfo fileInfo = hsManager.getFileInfo(jobId);
- Job result = null;
- if (fileInfo != null) {
- result = loadedJobCache.get(jobId);
- if (result == null) {
- result = loadJob(fileInfo);
- } else if(fileInfo.isDeleted()) {
- loadedJobCache.remove(jobId);
- result = null;
- }
+ retVal = loadedJobCache.getUnchecked(jobId);
+ } catch (UncheckedExecutionException e) {
+ if (e.getCause() instanceof HSFileRuntimeException) {
+ LOG.error(e.getCause().getMessage());
+ return null;
} else {
- loadedJobCache.remove(jobId);
+ throw new YarnRuntimeException(e.getCause());
}
- return result;
- } catch (IOException e) {
- throw new YarnRuntimeException(e);
}
+ return retVal;
}
@Override
@@ -243,4 +297,14 @@ public class CachedHistoryStorage extends AbstractService implements
}
return allJobs;
}
+
+ @VisibleForTesting
+ public boolean getUseLoadedTasksCache() {
+ return useLoadedTasksCache;
+ }
+
+ @VisibleForTesting
+ public int getLoadedTasksCacheSize() {
+ return loadedTasksCacheSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f72da7e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
index de0de7d..936c772 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistory.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
+import com.google.common.cache.Cache;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -35,16 +36,27 @@ import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-
+import static junit.framework.TestCase.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
public class TestJobHistory {
@@ -58,13 +70,15 @@ public class TestJobHistory {
Configuration conf = new Configuration();
// Set the cache size to 2
- conf.set(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, "2");
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 2);
jobHistory.init(conf);
jobHistory.start();
CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
.getHistoryStorage());
+ assertFalse(storage.getUseLoadedTasksCache());
+
Job[] jobs = new Job[3];
JobId[] jobIds = new JobId[3];
@@ -84,14 +98,13 @@ public class TestJobHistory {
storage.getFullJob(jobs[i].getID());
}
- Map<JobId, Job> jobCache = storage.getLoadedJobCache();
- // job0 should have been purged since cache size is 2
- assertFalse(jobCache.containsKey(jobs[0].getID()));
- assertTrue(jobCache.containsKey(jobs[1].getID())
- && jobCache.containsKey(jobs[2].getID()));
+ Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
+ // Verify some jobs are stored in the cache. Hard to predict eviction
+ // in Guava version.
+ assertTrue(jobCache.size() > 0);
// Setting cache size to 3
- conf.set(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, "3");
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 3);
doReturn(conf).when(storage).createConf();
when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1])
@@ -105,9 +118,223 @@ public class TestJobHistory {
jobCache = storage.getLoadedJobCache();
- // All three jobs should be in cache since its size is now 3
- for (int i = 0; i < 3; i++) {
- assertTrue(jobCache.containsKey(jobs[i].getID()));
+ // Verify some jobs are stored in the cache. Hard to predict eviction
+ // in Guava version.
+ assertTrue(jobCache.size() > 0);
+ }
+
+ @Test
+ public void testTasksCacheLimit() throws Exception {
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+
+ Configuration conf = new Configuration();
+ // Set the cache threshold to 50 tasks
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertTrue(storage.getUseLoadedTasksCache());
+ assertEquals(storage.getLoadedTasksCacheSize(), 50);
+
+ // Create a bunch of smaller jobs (<< 50 tasks)
+ Job[] jobs = new Job[10];
+ JobId[] jobIds = new JobId[10];
+ for (int i = 0; i < jobs.length; i++) {
+ jobs[i] = mock(Job.class);
+ jobIds[i] = mock(JobId.class);
+ when(jobs[i].getID()).thenReturn(jobIds[i]);
+ when(jobs[i].getTotalMaps()).thenReturn(10);
+ when(jobs[i].getTotalReduces()).thenReturn(2);
+ }
+
+ // Create some large jobs that forces task-based cache flushing
+ Job[] lgJobs = new Job[3];
+ JobId[] lgJobIds = new JobId[3];
+ for (int i = 0; i < lgJobs.length; i++) {
+ lgJobs[i] = mock(Job.class);
+ lgJobIds[i] = mock(JobId.class);
+ when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
+ when(lgJobs[i].getTotalMaps()).thenReturn(2000);
+ when(lgJobs[i].getTotalReduces()).thenReturn(10);
+ }
+
+ HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
+ when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
+ when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1])
+ .thenReturn(jobs[2]).thenReturn(jobs[3]).thenReturn(jobs[4])
+ .thenReturn(jobs[5]).thenReturn(jobs[6]).thenReturn(jobs[7])
+ .thenReturn(jobs[8]).thenReturn(jobs[9]).thenReturn(lgJobs[0])
+ .thenReturn(lgJobs[1]).thenReturn(lgJobs[2]);
+
+ // getFullJob will put the job in the cache if it isn't there
+ Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
+ for (int i = 0; i < jobs.length; i++) {
+ storage.getFullJob(jobs[i].getID());
+ }
+ long prevSize = jobCache.size();
+
+ // Fill the cache with some larger jobs and verify the cache
+ // gets reduced in size.
+ for (int i = 0; i < lgJobs.length; i++) {
+ storage.getFullJob(lgJobs[i].getID());
+ }
+ assertTrue(jobCache.size() < prevSize);
+ }
+
+ @Test
+ public void testJobCacheLimitLargerThanMax() throws Exception {
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ JobHistory jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+
+ Configuration conf = new Configuration();
+ // Set the cache threshold to 50 tasks
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 500);
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertTrue(storage.getUseLoadedTasksCache());
+ assertEquals(storage.getLoadedTasksCacheSize(), 500);
+
+ // Create a bunch of large jobs (>> 50 tasks)
+ Job[] lgJobs = new Job[10];
+ JobId[] lgJobIds = new JobId[10];
+ for (int i = 0; i < lgJobs.length; i++) {
+ lgJobs[i] = mock(Job.class);
+ lgJobIds[i] = mock(JobId.class);
+ when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
+ when(lgJobs[i].getTotalMaps()).thenReturn(700);
+ when(lgJobs[i].getTotalReduces()).thenReturn(50);
+ }
+
+ HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
+ when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
+ when(fileInfo.loadJob()).thenReturn(lgJobs[0]).thenReturn(lgJobs[1])
+ .thenReturn(lgJobs[2]).thenReturn(lgJobs[3]).thenReturn(lgJobs[4])
+ .thenReturn(lgJobs[5]).thenReturn(lgJobs[6]).thenReturn(lgJobs[7])
+ .thenReturn(lgJobs[8]).thenReturn(lgJobs[9]);
+
+ // getFullJob will put the job in the cache if it isn't there
+ Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
+ long[] cacheSize = new long[10];
+ for (int i = 0; i < lgJobs.length; i++) {
+ storage.getFullJob(lgJobs[i].getID());
+ assertTrue(jobCache.size() > 0);
+ }
+ }
+
+ @Test
+ public void testLoadedTasksEmptyConfiguration() {
+ Configuration conf = new Configuration();
+ conf.set(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, "");
+
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ JobHistory jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertFalse(storage.getUseLoadedTasksCache());
+ }
+
+ @Test
+ public void testLoadedTasksZeroConfiguration() {
+ Configuration conf = new Configuration();
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 0);
+
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ JobHistory jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertTrue(storage.getUseLoadedTasksCache());
+ assertEquals(storage.getLoadedTasksCacheSize(), 1);
+ }
+
+ @Test
+ public void testLoadedTasksNegativeConfiguration() {
+ Configuration conf = new Configuration();
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, -1);
+
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ JobHistory jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertTrue(storage.getUseLoadedTasksCache());
+ assertEquals(storage.getLoadedTasksCacheSize(), 1);
+ }
+
+ @Test
+ public void testLoadJobErrorCases() throws IOException {
+ HistoryFileManager historyManager = mock(HistoryFileManager.class);
+ jobHistory = spy(new JobHistory());
+ doReturn(historyManager).when(jobHistory).createHistoryFileManager();
+
+ Configuration conf = new Configuration();
+ // Set the cache threshold to 50 tasks
+ conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
+ jobHistory.init(conf);
+ jobHistory.start();
+
+ CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory
+ .getHistoryStorage());
+
+ assertTrue(storage.getUseLoadedTasksCache());
+ assertEquals(storage.getLoadedTasksCacheSize(), 50);
+
+ // Create jobs for bad fileInfo results
+ Job[] jobs = new Job[4];
+ JobId[] jobIds = new JobId[4];
+ for (int i = 0; i < jobs.length; i++) {
+ jobs[i] = mock(Job.class);
+ jobIds[i] = mock(JobId.class);
+ when(jobs[i].getID()).thenReturn(jobIds[i]);
+ when(jobs[i].getTotalMaps()).thenReturn(10);
+ when(jobs[i].getTotalReduces()).thenReturn(2);
+ }
+
+ HistoryFileInfo loadJobException = mock(HistoryFileInfo.class);
+ when(loadJobException.loadJob()).thenThrow(new IOException("History file not found"));
+ when(historyManager.getFileInfo(jobIds[0])).thenThrow(new IOException(""));
+ when(historyManager.getFileInfo(jobIds[1])).thenReturn(null);
+ when(historyManager.getFileInfo(jobIds[2])).thenReturn(loadJobException);
+
+ try {
+ storage.getFullJob(jobIds[0]);
+ fail("Did not get expected YarnRuntimeException for getFileInfo() throwing IOException");
+ } catch (YarnRuntimeException e) {
+ // Expected
+ }
+
+ // fileInfo==null should return null
+ Job job = storage.getFullJob(jobIds[1]);
+ assertNull(job);
+
+ try {
+ storage.getFullJob(jobIds[2]);
+ fail("Did not get expected YarnRuntimeException for fileInfo.loadJob() throwing IOException");
+ } catch (YarnRuntimeException e) {
+ // Expected
}
}
[20/50] [abbrv] hadoop git commit: HDFS-7964. Add support for async
edit logging. Contributed by Daryn Sharp.
Posted by ar...@apache.org.
HDFS-7964. Add support for async edit logging. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21517168
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21517168
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21517168
Branch: refs/heads/HDFS-1312
Commit: 2151716832ad14932dd65b1a4e47e64d8d6cd767
Parents: 0fa54d4
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Feb 29 15:34:43 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon Feb 29 15:34:43 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../bkjournal/TestBookKeeperAsHASharedDir.java | 46 ++-
.../src/test/resources/log4j.properties | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +
.../hadoop/hdfs/server/namenode/FSEditLog.java | 109 ++++---
.../hdfs/server/namenode/FSEditLogAsync.java | 322 +++++++++++++++++++
.../hdfs/server/namenode/FSEditLogOp.java | 213 ++++++------
.../hdfs/server/namenode/FSEditLogOpCodes.java | 108 ++++---
.../hadoop/hdfs/server/namenode/FSImage.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 1 -
.../namenode/metrics/NameNodeMetrics.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 3 +
.../hdfs/server/namenode/TestAuditLogs.java | 15 +-
.../hdfs/server/namenode/TestEditLog.java | 59 +++-
.../server/namenode/TestEditLogAutoroll.java | 26 ++
.../namenode/TestEditLogJournalFailures.java | 35 +-
.../hdfs/server/namenode/TestEditLogRace.java | 144 +++++----
.../server/namenode/TestFSEditLogLoader.java | 37 ++-
.../server/namenode/TestNameNodeRecovery.java | 31 +-
.../server/namenode/ha/TestEditLogTailer.java | 39 ++-
21 files changed, 904 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3d57efa..c3ea5ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1037,6 +1037,8 @@ Release 2.9.0 - UNRELEASED
HDFS-9754. Avoid unnecessary getBlockCollection calls in BlockManager.
(jing9)
+ HDFS-7964. Add support for async edit logging. (Daryn Sharp)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
index 5611bb8..ff8c00d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
@@ -24,6 +24,9 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.AfterClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
@@ -56,11 +59,14 @@ import org.apache.commons.logging.LogFactory;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
/**
* Integration test to ensure that the BookKeeper JournalManager
* works for HDFS Namenode HA
*/
+@RunWith(Parameterized.class)
public class TestBookKeeperAsHASharedDir {
static final Log LOG = LogFactory.getLog(TestBookKeeperAsHASharedDir.class);
@@ -69,6 +75,27 @@ public class TestBookKeeperAsHASharedDir {
private static final String TEST_FILE_DATA = "HA BookKeeperJournalManager";
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestBookKeeperAsHASharedDir(Boolean async) {
+ useAsyncEditLog = async;
+ }
+
+ private static Configuration getConf() {
+ Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
+ return conf;
+ }
+
@BeforeClass
public static void setupBookkeeper() throws Exception {
bkutil = new BKJMUtil(numBookies);
@@ -92,8 +119,7 @@ public class TestBookKeeperAsHASharedDir {
public void testFailoverWithBK() throws Exception {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ Configuration conf = getConf();
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
BKJMUtil.createJournalURI("/hotfailover").toString());
BKJMUtil.addJournalManagerDefinition(conf);
@@ -144,8 +170,7 @@ public class TestBookKeeperAsHASharedDir {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ Configuration conf = getConf();
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
BKJMUtil.createJournalURI("/hotfailoverWithFail").toString());
conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE,
@@ -221,8 +246,7 @@ public class TestBookKeeperAsHASharedDir {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ Configuration conf = getConf();
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
BKJMUtil.createJournalURI("/hotfailoverMultiple").toString());
BKJMUtil.addJournalManagerDefinition(conf);
@@ -245,7 +269,9 @@ public class TestBookKeeperAsHASharedDir {
fs = cluster.getFileSystem(0); // get the older active server.
try {
- fs.delete(p1, true);
+ System.out.println("DMS: > *************");
+ boolean foo = fs.delete(p1, true);
+ System.out.println("DMS: < ************* "+foo);
fail("Log update on older active should cause it to exit");
} catch (RemoteException re) {
assertTrue(re.getClassName().contains("ExitException"));
@@ -267,9 +293,8 @@ public class TestBookKeeperAsHASharedDir {
public void testInitializeBKSharedEdits() throws Exception {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
HAUtil.setAllowStandbyReads(conf, true);
- conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
MiniDFSNNTopology topology = MiniDFSNNTopology.simpleHATopology();
cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology)
@@ -358,8 +383,7 @@ public class TestBookKeeperAsHASharedDir {
public void testNameNodeMultipleSwitchesUsingBKJM() throws Exception {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ Configuration conf = getConf();
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil
.createJournalURI("/correctEditLogSelection").toString());
BKJMUtil.addJournalManagerDefinition(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
index 93c22f7..52aac43 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
@@ -26,7 +26,7 @@
# Format is "<default threshold> (, <appender>)+
# DEFAULT: console appender only
-log4j.rootLogger=OFF, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
# Example with rolling log file
#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bfb6203..9c06e29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -291,7 +291,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH = "dfs.namenode.edits.noeditlogchannelflush";
public static final boolean DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH_DEFAULT = false;
-
+
+ public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
+ "dfs.namenode.edits.asynclogging";
+ public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
index 36053f7..d36e0b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
@@ -142,6 +142,10 @@ public class BackupNode extends NameNode {
@Override // NameNode
protected void initialize(Configuration conf) throws IOException {
+ // async edit logs are incompatible with backup node due to race
+ // conditions resulting from laxer synchronization
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, false);
+
// Trash is disabled in BackupNameNode,
// but should be turned back on if it ever becomes active.
conf.setLong(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index c8986dc..809d9e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -78,7 +78,9 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOldOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameSnapshotOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenewDelegationTokenOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeFinalizeOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeStartOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetAclOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV1Op;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV2Op;
@@ -115,7 +117,7 @@ import com.google.common.collect.Lists;
@InterfaceStability.Evolving
public class FSEditLog implements LogsPurgeable {
- static final Log LOG = LogFactory.getLog(FSEditLog.class);
+ public static final Log LOG = LogFactory.getLog(FSEditLog.class);
/**
* State machine for edit log.
@@ -178,17 +180,11 @@ public class FSEditLog implements LogsPurgeable {
private final NNStorage storage;
private final Configuration conf;
-
+
private final List<URI> editsDirs;
- private final ThreadLocal<OpInstanceCache> cache =
- new ThreadLocal<OpInstanceCache>() {
- @Override
- protected OpInstanceCache initialValue() {
- return new OpInstanceCache();
- }
- };
-
+ protected final OpInstanceCache cache = new OpInstanceCache();
+
/**
* The edit directories that are shared between primary and secondary.
*/
@@ -217,6 +213,17 @@ public class FSEditLog implements LogsPurgeable {
}
};
+ static FSEditLog newInstance(Configuration conf, NNStorage storage,
+ List<URI> editsDirs) {
+ boolean asyncEditLogging = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT);
+ LOG.info("Edit logging is async:" + asyncEditLogging);
+ return asyncEditLogging
+ ? new FSEditLogAsync(conf, storage, editsDirs)
+ : new FSEditLog(conf, storage, editsDirs);
+ }
+
/**
* Constructor for FSEditLog. Underlying journals are constructed, but
* no streams are opened until open() is called.
@@ -423,33 +430,35 @@ public class FSEditLog implements LogsPurgeable {
// wait if an automatic sync is scheduled
waitIfAutoSyncScheduled();
-
- long start = beginTransaction();
- op.setTransactionId(txid);
-
- try {
- editLogStream.write(op);
- } catch (IOException ex) {
- // All journals failed, it is handled in logSync.
- } finally {
- op.reset();
- }
- endTransaction(start);
-
// check if it is time to schedule an automatic sync
- needsSync = shouldForceSync();
+ needsSync = doEditTransaction(op);
if (needsSync) {
isAutoSyncScheduled = true;
}
}
-
+
// Sync the log if an automatic sync is required.
if (needsSync) {
logSync();
}
}
+ synchronized boolean doEditTransaction(final FSEditLogOp op) {
+ long start = beginTransaction();
+ op.setTransactionId(txid);
+
+ try {
+ editLogStream.write(op);
+ } catch (IOException ex) {
+ // All journals failed, it is handled in logSync.
+ } finally {
+ op.reset();
+ }
+ endTransaction(start);
+ return shouldForceSync();
+ }
+
/**
* Wait if an automatic sync is scheduled
*/
@@ -544,15 +553,10 @@ public class FSEditLog implements LogsPurgeable {
* else more operations can start writing while this is in progress.
*/
void logSyncAll() {
- // Record the most recent transaction ID as our own id
- synchronized (this) {
- TransactionId id = myTransactionId.get();
- id.txid = txid;
- }
- // Then make sure we're synced up to this point
- logSync();
+ // Make sure we're synced up to the most recent transaction ID.
+ logSync(getLastWrittenTxId());
}
-
+
/**
* Sync all modifications done by this thread.
*
@@ -582,12 +586,14 @@ public class FSEditLog implements LogsPurgeable {
* waitForSyncToFinish() before assuming they are running alone.
*/
public void logSync() {
- long syncStart = 0;
+ // Fetch the transactionId of this thread.
+ logSync(myTransactionId.get().txid);
+ }
- // Fetch the transactionId of this thread.
- long mytxid = myTransactionId.get().txid;
-
+ protected void logSync(long mytxid) {
+ long syncStart = 0;
boolean sync = false;
+ long editsBatchedInSync = 0;
try {
EditLogOutputStream logStream = null;
synchronized (this) {
@@ -606,19 +612,17 @@ public class FSEditLog implements LogsPurgeable {
// If this transaction was already flushed, then nothing to do
//
if (mytxid <= synctxid) {
- numTransactionsBatchedInSync++;
- if (metrics != null) {
- // Metrics is non-null only when used inside name node
- metrics.incrTransactionsBatchedInSync();
- }
return;
}
-
- // now, this thread will do the sync
+
+ // now, this thread will do the sync. track if other edits were
+ // included in the sync - ie. batched. if this is the only edit
+ // synced then the batched count is 0
+ editsBatchedInSync = txid - synctxid - 1;
syncStart = txid;
isSyncRunning = true;
sync = true;
-
+
// swap buffers
try {
if (journalSet.isEmpty()) {
@@ -667,6 +671,8 @@ public class FSEditLog implements LogsPurgeable {
if (metrics != null) { // Metrics non-null only when used inside name node
metrics.addSync(elapsed);
+ metrics.incrTransactionsBatchedInSync(editsBatchedInSync);
+ numTransactionsBatchedInSync += editsBatchedInSync;
}
} finally {
@@ -1138,13 +1144,13 @@ public class FSEditLog implements LogsPurgeable {
}
void logStartRollingUpgrade(long startTime) {
- RollingUpgradeOp op = RollingUpgradeOp.getStartInstance(cache.get());
+ RollingUpgradeStartOp op = RollingUpgradeStartOp.getInstance(cache.get());
op.setTime(startTime);
logEdit(op);
}
void logFinalizeRollingUpgrade(long finalizeTime) {
- RollingUpgradeOp op = RollingUpgradeOp.getFinalizeInstance(cache.get());
+ RollingUpgradeOp op = RollingUpgradeFinalizeOp.getInstance(cache.get());
op.setTime(finalizeTime);
logEdit(op);
}
@@ -1313,8 +1319,9 @@ public class FSEditLog implements LogsPurgeable {
if (writeEndTxn) {
logEdit(LogSegmentOp.getInstance(cache.get(),
FSEditLogOpCodes.OP_END_LOG_SEGMENT));
- logSync();
}
+ // always sync to ensure all edits are flushed.
+ logSyncAll();
printStatistics(true);
@@ -1701,6 +1708,12 @@ public class FSEditLog implements LogsPurgeable {
}
}
+ @VisibleForTesting
+ // needed by async impl to restart thread when edit log is replaced by a
+ // spy because a spy is a shallow copy
+ public void restart() {
+ }
+
/**
* Return total number of syncs happened on this edit log.
* @return long - count
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
new file mode 100644
index 0000000..c14a310
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -0,0 +1,322 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.util.ExitUtil;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+class FSEditLogAsync extends FSEditLog implements Runnable {
+ static final Log LOG = LogFactory.getLog(FSEditLog.class);
+
+ // use separate mutex to avoid possible deadlock when stopping the thread.
+ private final Object syncThreadLock = new Object();
+ private Thread syncThread;
+ private static ThreadLocal<Edit> threadEdit = new ThreadLocal<Edit>();
+
+ // requires concurrent access from caller threads and syncing thread.
+ private final BlockingQueue<Edit> editPendingQ =
+ new ArrayBlockingQueue<Edit>(4096);
+
+ // only accessed by syncing thread so no synchronization required.
+ // queue is unbounded because it's effectively limited by the size
+ // of the edit log buffer - ie. a sync will eventually be forced.
+ private final Deque<Edit> syncWaitQ = new ArrayDeque<Edit>();
+
+ FSEditLogAsync(Configuration conf, NNStorage storage, List<URI> editsDirs) {
+ super(conf, storage, editsDirs);
+ // op instances cannot be shared due to queuing for background thread.
+ cache.disableCache();
+ }
+
+ private boolean isSyncThreadAlive() {
+ synchronized(syncThreadLock) {
+ return syncThread != null && syncThread.isAlive();
+ }
+ }
+
+ private void startSyncThread() {
+ synchronized(syncThreadLock) {
+ if (!isSyncThreadAlive()) {
+ syncThread = new Thread(this, this.getClass().getSimpleName());
+ syncThread.start();
+ }
+ }
+ }
+
+ private void stopSyncThread() {
+ synchronized(syncThreadLock) {
+ if (syncThread != null) {
+ try {
+ syncThread.interrupt();
+ syncThread.join();
+ } catch (InterruptedException e) {
+ // we're quitting anyway.
+ } finally {
+ syncThread = null;
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ @Override
+ public void restart() {
+ stopSyncThread();
+ startSyncThread();
+ }
+
+ @Override
+ void openForWrite(int layoutVersion) throws IOException {
+ try {
+ startSyncThread();
+ super.openForWrite(layoutVersion);
+ } catch (IOException ioe) {
+ stopSyncThread();
+ throw ioe;
+ }
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ stopSyncThread();
+ }
+
+ @Override
+ void logEdit(final FSEditLogOp op) {
+ Edit edit = getEditInstance(op);
+ threadEdit.set(edit);
+ enqueueEdit(edit);
+ }
+
+ @Override
+ public void logSync() {
+ Edit edit = threadEdit.get();
+ if (edit != null) {
+ // do NOT remove to avoid expunge & rehash penalties.
+ threadEdit.set(null);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("logSync " + edit);
+ }
+ edit.logSyncWait();
+ }
+ }
+
+ @Override
+ public void logSyncAll() {
+ // doesn't actually log anything, just ensures that the queues are
+ // drained when it returns.
+ Edit edit = new SyncEdit(this, null){
+ @Override
+ public boolean logEdit() {
+ return true;
+ }
+ };
+ enqueueEdit(edit);
+ edit.logSyncWait();
+ }
+
+ private void enqueueEdit(Edit edit) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("logEdit " + edit);
+ }
+ try {
+ if (!editPendingQ.offer(edit, 1, TimeUnit.SECONDS)) {
+ Preconditions.checkState(
+ isSyncThreadAlive(), "sync thread is not alive");
+ editPendingQ.put(edit);
+ }
+ } catch (Throwable t) {
+ // should never happen! failure to enqueue an edit is fatal
+ terminate(t);
+ }
+ }
+
+ private Edit dequeueEdit() throws InterruptedException {
+ // only block for next edit if no pending syncs.
+ return syncWaitQ.isEmpty() ? editPendingQ.take() : editPendingQ.poll();
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ boolean doSync;
+ Edit edit = dequeueEdit();
+ if (edit != null) {
+ // sync if requested by edit log.
+ doSync = edit.logEdit();
+ syncWaitQ.add(edit);
+ } else {
+ // sync when editq runs dry, but have edits pending a sync.
+ doSync = !syncWaitQ.isEmpty();
+ }
+ if (doSync) {
+ // normally edit log exceptions cause the NN to terminate, but tests
+ // relying on ExitUtil.terminate need to see the exception.
+ RuntimeException syncEx = null;
+ try {
+ logSync(getLastWrittenTxId());
+ } catch (RuntimeException ex) {
+ syncEx = ex;
+ }
+ while ((edit = syncWaitQ.poll()) != null) {
+ edit.logSyncNotify(syncEx);
+ }
+ }
+ }
+ } catch (InterruptedException ie) {
+ LOG.info(Thread.currentThread().getName() + " was interrupted, exiting");
+ } catch (Throwable t) {
+ terminate(t);
+ }
+ }
+
+ private void terminate(Throwable t) {
+ String message = "Exception while edit logging: "+t.getMessage();
+ LOG.fatal(message, t);
+ ExitUtil.terminate(1, message);
+ }
+
+ private Edit getEditInstance(FSEditLogOp op) {
+ final Edit edit;
+ final Server.Call rpcCall = Server.getCurCall().get();
+ // only rpc calls not explicitly sync'ed on the log will be async.
+ if (rpcCall != null && !Thread.holdsLock(this)) {
+ edit = new RpcEdit(this, op, rpcCall);
+ } else {
+ edit = new SyncEdit(this, op);
+ }
+ return edit;
+ }
+
+ private abstract static class Edit {
+ final FSEditLog log;
+ final FSEditLogOp op;
+
+ Edit(FSEditLog log, FSEditLogOp op) {
+ this.log = log;
+ this.op = op;
+ }
+
+ // return whether edit log wants to sync.
+ boolean logEdit() {
+ return log.doEditTransaction(op);
+ }
+
+ // wait for background thread to finish syncing.
+ abstract void logSyncWait();
+ // wake up the thread in logSyncWait.
+ abstract void logSyncNotify(RuntimeException ex);
+ }
+
+ // the calling thread is synchronously waiting for the edit to complete.
+ private static class SyncEdit extends Edit {
+ private final Object lock;
+ private boolean done = false;
+ private RuntimeException syncEx;
+
+ SyncEdit(FSEditLog log, FSEditLogOp op) {
+ super(log, op);
+ // if the log is already sync'ed (ex. log rolling), must wait on it to
+ // avoid deadlock with sync thread. the fsn lock protects against
+ // logging during a roll. else lock on this object to avoid sync
+ // contention on edit log.
+ lock = Thread.holdsLock(log) ? log : this;
+ }
+
+ @Override
+ public void logSyncWait() {
+ synchronized(lock) {
+ while (!done) {
+ try {
+ lock.wait(10);
+ } catch (InterruptedException e) {}
+ }
+ // only needed by tests that rely on ExitUtil.terminate() since
+ // normally exceptions terminate the NN.
+ if (syncEx != null) {
+ syncEx.fillInStackTrace();
+ throw syncEx;
+ }
+ }
+ }
+
+ @Override
+ public void logSyncNotify(RuntimeException ex) {
+ synchronized(lock) {
+ done = true;
+ syncEx = ex;
+ lock.notifyAll();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "["+getClass().getSimpleName()+" op:"+op+"]";
+ }
+ }
+
+ // the calling rpc thread will return immediately from logSync but the
+ // rpc response will not be sent until the edit is durable.
+ private static class RpcEdit extends Edit {
+ private final Server.Call call;
+
+ RpcEdit(FSEditLog log, FSEditLogOp op, Server.Call call) {
+ super(log, op);
+ this.call = call;
+ call.postponeResponse();
+ }
+
+ @Override
+ public void logSyncWait() {
+ // logSync is a no-op to immediately free up rpc handlers. the
+ // response is sent when the sync thread calls syncNotify.
+ }
+
+ @Override
+ public void logSyncNotify(RuntimeException syncEx) {
+ try {
+ if (syncEx == null) {
+ call.sendResponse();
+ } else {
+ call.abortResponse(syncEx);
+ }
+ } catch (Exception e) {} // don't care if not sent.
+ }
+
+ @Override
+ public String toString() {
+ return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index a8389f0..c4e1a78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -147,6 +147,55 @@ public abstract class FSEditLogOp {
byte[] rpcClientId;
int rpcCallId;
+ public static class OpInstanceCache {
+ private static ThreadLocal<OpInstanceCacheMap> cache =
+ new ThreadLocal<OpInstanceCacheMap>() {
+ @Override
+ protected OpInstanceCacheMap initialValue() {
+ return new OpInstanceCacheMap();
+ }
+ };
+
+ @SuppressWarnings("serial")
+ static final class OpInstanceCacheMap extends
+ EnumMap<FSEditLogOpCodes, FSEditLogOp> {
+ OpInstanceCacheMap() {
+ super(FSEditLogOpCodes.class);
+ for (FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) {
+ put(opCode, newInstance(opCode));
+ }
+ }
+ }
+
+ private boolean useCache = true;
+
+ void disableCache() {
+ useCache = false;
+ }
+
+ public OpInstanceCache get() {
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends FSEditLogOp> T get(FSEditLogOpCodes opCode) {
+ return useCache ? (T)cache.get().get(opCode) : (T)newInstance(opCode);
+ }
+
+ private static FSEditLogOp newInstance(FSEditLogOpCodes opCode) {
+ FSEditLogOp instance = null;
+ Class<? extends FSEditLogOp> clazz = opCode.getOpClass();
+ if (clazz != null) {
+ try {
+ instance = clazz.newInstance();
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to instantiate "+opCode, ex);
+ }
+ }
+ return instance;
+ }
+ }
+
final void reset() {
txid = HdfsServerConstants.INVALID_TXID;
rpcClientId = RpcConstants.DUMMY_CLIENT_ID;
@@ -156,70 +205,6 @@ public abstract class FSEditLogOp {
abstract void resetSubFields();
- final public static class OpInstanceCache {
- private final EnumMap<FSEditLogOpCodes, FSEditLogOp> inst =
- new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
-
- public OpInstanceCache() {
- inst.put(OP_ADD, new AddOp());
- inst.put(OP_CLOSE, new CloseOp());
- inst.put(OP_SET_REPLICATION, new SetReplicationOp());
- inst.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
- inst.put(OP_RENAME_OLD, new RenameOldOp());
- inst.put(OP_DELETE, new DeleteOp());
- inst.put(OP_MKDIR, new MkdirOp());
- inst.put(OP_SET_GENSTAMP_V1, new SetGenstampV1Op());
- inst.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
- inst.put(OP_SET_OWNER, new SetOwnerOp());
- inst.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
- inst.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
- inst.put(OP_SET_QUOTA, new SetQuotaOp());
- inst.put(OP_TIMES, new TimesOp());
- inst.put(OP_SYMLINK, new SymlinkOp());
- inst.put(OP_RENAME, new RenameOp());
- inst.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
- inst.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
- inst.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
- inst.put(OP_CANCEL_DELEGATION_TOKEN, new CancelDelegationTokenOp());
- inst.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
- inst.put(OP_START_LOG_SEGMENT, new LogSegmentOp(OP_START_LOG_SEGMENT));
- inst.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT));
- inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
- inst.put(OP_TRUNCATE, new TruncateOp());
-
- inst.put(OP_ALLOW_SNAPSHOT, new AllowSnapshotOp());
- inst.put(OP_DISALLOW_SNAPSHOT, new DisallowSnapshotOp());
- inst.put(OP_CREATE_SNAPSHOT, new CreateSnapshotOp());
- inst.put(OP_DELETE_SNAPSHOT, new DeleteSnapshotOp());
- inst.put(OP_RENAME_SNAPSHOT, new RenameSnapshotOp());
- inst.put(OP_SET_GENSTAMP_V2, new SetGenstampV2Op());
- inst.put(OP_ALLOCATE_BLOCK_ID, new AllocateBlockIdOp());
-
- inst.put(OP_ADD_CACHE_DIRECTIVE, new AddCacheDirectiveInfoOp());
- inst.put(OP_MODIFY_CACHE_DIRECTIVE, new ModifyCacheDirectiveInfoOp());
- inst.put(OP_REMOVE_CACHE_DIRECTIVE, new RemoveCacheDirectiveInfoOp());
- inst.put(OP_ADD_CACHE_POOL, new AddCachePoolOp());
- inst.put(OP_MODIFY_CACHE_POOL, new ModifyCachePoolOp());
- inst.put(OP_REMOVE_CACHE_POOL, new RemoveCachePoolOp());
-
- inst.put(OP_ADD_BLOCK, new AddBlockOp());
- inst.put(OP_SET_ACL, new SetAclOp());
- inst.put(OP_ROLLING_UPGRADE_START, new RollingUpgradeOp(
- OP_ROLLING_UPGRADE_START, "start"));
- inst.put(OP_ROLLING_UPGRADE_FINALIZE, new RollingUpgradeOp(
- OP_ROLLING_UPGRADE_FINALIZE, "finalize"));
- inst.put(OP_SET_XATTR, new SetXAttrOp());
- inst.put(OP_REMOVE_XATTR, new RemoveXAttrOp());
- inst.put(OP_SET_STORAGE_POLICY, new SetStoragePolicyOp());
- inst.put(OP_APPEND, new AppendOp());
- inst.put(OP_SET_QUOTA_BY_STORAGETYPE, new SetQuotaByStorageTypeOp());
- }
-
- public FSEditLogOp get(FSEditLogOpCodes opcode) {
- return inst.get(opcode);
- }
- }
-
private static ImmutableMap<String, FsAction> fsActionMap() {
ImmutableMap.Builder<String, FsAction> b = ImmutableMap.builder();
for (FsAction v : FsAction.values())
@@ -774,7 +759,7 @@ public abstract class FSEditLogOp {
* {@link ClientProtocol#append}
*/
static class AddOp extends AddCloseOp {
- private AddOp() {
+ AddOp() {
super(OP_ADD);
}
@@ -802,7 +787,7 @@ public abstract class FSEditLogOp {
* finally log an AddOp.
*/
static class CloseOp extends AddCloseOp {
- private CloseOp() {
+ CloseOp() {
super(OP_CLOSE);
}
@@ -830,7 +815,7 @@ public abstract class FSEditLogOp {
String clientMachine;
boolean newBlock;
- private AppendOp() {
+ AppendOp() {
super(OP_APPEND);
}
@@ -920,7 +905,7 @@ public abstract class FSEditLogOp {
private Block penultimateBlock;
private Block lastBlock;
- private AddBlockOp() {
+ AddBlockOp() {
super(OP_ADD_BLOCK);
}
@@ -1032,7 +1017,7 @@ public abstract class FSEditLogOp {
String path;
Block[] blocks;
- private UpdateBlocksOp() {
+ UpdateBlocksOp() {
super(OP_UPDATE_BLOCKS);
}
@@ -1125,7 +1110,7 @@ public abstract class FSEditLogOp {
String path;
short replication;
- private SetReplicationOp() {
+ SetReplicationOp() {
super(OP_SET_REPLICATION);
}
@@ -1204,7 +1189,7 @@ public abstract class FSEditLogOp {
long timestamp;
final static public int MAX_CONCAT_SRC = 1024 * 1024;
- private ConcatDeleteOp() {
+ ConcatDeleteOp() {
super(OP_CONCAT_DELETE);
}
@@ -1362,7 +1347,7 @@ public abstract class FSEditLogOp {
String dst;
long timestamp;
- private RenameOldOp() {
+ RenameOldOp() {
super(OP_RENAME_OLD);
}
@@ -1474,7 +1459,7 @@ public abstract class FSEditLogOp {
String path;
long timestamp;
- private DeleteOp() {
+ DeleteOp() {
super(OP_DELETE);
}
@@ -1575,7 +1560,7 @@ public abstract class FSEditLogOp {
List<AclEntry> aclEntries;
List<XAttr> xAttrs;
- private MkdirOp() {
+ MkdirOp() {
super(OP_MKDIR);
}
@@ -1748,7 +1733,7 @@ public abstract class FSEditLogOp {
static class SetGenstampV1Op extends FSEditLogOp {
long genStampV1;
- private SetGenstampV1Op() {
+ SetGenstampV1Op() {
super(OP_SET_GENSTAMP_V1);
}
@@ -1806,7 +1791,7 @@ public abstract class FSEditLogOp {
static class SetGenstampV2Op extends FSEditLogOp {
long genStampV2;
- private SetGenstampV2Op() {
+ SetGenstampV2Op() {
super(OP_SET_GENSTAMP_V2);
}
@@ -1864,7 +1849,7 @@ public abstract class FSEditLogOp {
static class AllocateBlockIdOp extends FSEditLogOp {
long blockId;
- private AllocateBlockIdOp() {
+ AllocateBlockIdOp() {
super(OP_ALLOCATE_BLOCK_ID);
}
@@ -1923,7 +1908,7 @@ public abstract class FSEditLogOp {
String src;
FsPermission permissions;
- private SetPermissionsOp() {
+ SetPermissionsOp() {
super(OP_SET_PERMISSIONS);
}
@@ -1996,7 +1981,7 @@ public abstract class FSEditLogOp {
String username;
String groupname;
- private SetOwnerOp() {
+ SetOwnerOp() {
super(OP_SET_OWNER);
}
@@ -2083,7 +2068,7 @@ public abstract class FSEditLogOp {
String src;
long nsQuota;
- private SetNSQuotaOp() {
+ SetNSQuotaOp() {
super(OP_SET_NS_QUOTA);
}
@@ -2141,7 +2126,7 @@ public abstract class FSEditLogOp {
static class ClearNSQuotaOp extends FSEditLogOp {
String src;
- private ClearNSQuotaOp() {
+ ClearNSQuotaOp() {
super(OP_CLEAR_NS_QUOTA);
}
@@ -2195,7 +2180,7 @@ public abstract class FSEditLogOp {
long nsQuota;
long dsQuota;
- private SetQuotaOp() {
+ SetQuotaOp() {
super(OP_SET_QUOTA);
}
@@ -2280,7 +2265,7 @@ public abstract class FSEditLogOp {
long dsQuota;
StorageType type;
- private SetQuotaByStorageTypeOp() {
+ SetQuotaByStorageTypeOp() {
super(OP_SET_QUOTA_BY_STORAGETYPE);
}
@@ -2363,7 +2348,7 @@ public abstract class FSEditLogOp {
long mtime;
long atime;
- private TimesOp() {
+ TimesOp() {
super(OP_TIMES);
}
@@ -2472,7 +2457,7 @@ public abstract class FSEditLogOp {
long atime;
PermissionStatus permissionStatus;
- private SymlinkOp() {
+ SymlinkOp() {
super(OP_SYMLINK);
}
@@ -2631,7 +2616,7 @@ public abstract class FSEditLogOp {
long timestamp;
Rename[] options;
- private RenameOp() {
+ RenameOp() {
super(OP_RENAME);
}
@@ -2796,7 +2781,7 @@ public abstract class FSEditLogOp {
long timestamp;
Block truncateBlock;
- private TruncateOp() {
+ TruncateOp() {
super(OP_TRUNCATE);
}
@@ -2929,7 +2914,7 @@ public abstract class FSEditLogOp {
String path;
String newHolder;
- private ReassignLeaseOp() {
+ ReassignLeaseOp() {
super(OP_REASSIGN_LEASE);
}
@@ -3011,7 +2996,7 @@ public abstract class FSEditLogOp {
DelegationTokenIdentifier token;
long expiryTime;
- private GetDelegationTokenOp() {
+ GetDelegationTokenOp() {
super(OP_GET_DELEGATION_TOKEN);
}
@@ -3090,7 +3075,7 @@ public abstract class FSEditLogOp {
DelegationTokenIdentifier token;
long expiryTime;
- private RenewDelegationTokenOp() {
+ RenewDelegationTokenOp() {
super(OP_RENEW_DELEGATION_TOKEN);
}
@@ -3168,7 +3153,7 @@ public abstract class FSEditLogOp {
static class CancelDelegationTokenOp extends FSEditLogOp {
DelegationTokenIdentifier token;
- private CancelDelegationTokenOp() {
+ CancelDelegationTokenOp() {
super(OP_CANCEL_DELEGATION_TOKEN);
}
@@ -3227,7 +3212,7 @@ public abstract class FSEditLogOp {
static class UpdateMasterKeyOp extends FSEditLogOp {
DelegationKey key;
- private UpdateMasterKeyOp() {
+ UpdateMasterKeyOp() {
super(OP_UPDATE_MASTER_KEY);
}
@@ -3332,8 +3317,20 @@ public abstract class FSEditLogOp {
}
}
+ static class StartLogSegmentOp extends LogSegmentOp {
+ StartLogSegmentOp() {
+ super(OP_START_LOG_SEGMENT);
+ }
+ }
+
+ static class EndLogSegmentOp extends LogSegmentOp {
+ EndLogSegmentOp() {
+ super(OP_END_LOG_SEGMENT);
+ }
+ }
+
static class InvalidOp extends FSEditLogOp {
- private InvalidOp() {
+ InvalidOp() {
super(OP_INVALID);
}
@@ -4144,7 +4141,7 @@ public abstract class FSEditLogOp {
List<XAttr> xAttrs;
String src;
- private RemoveXAttrOp() {
+ RemoveXAttrOp() {
super(OP_REMOVE_XATTR);
}
@@ -4197,7 +4194,7 @@ public abstract class FSEditLogOp {
List<XAttr> xAttrs;
String src;
- private SetXAttrOp() {
+ SetXAttrOp() {
super(OP_SET_XATTR);
}
@@ -4250,7 +4247,7 @@ public abstract class FSEditLogOp {
List<AclEntry> aclEntries = Lists.newArrayList();
String src;
- private SetAclOp() {
+ SetAclOp() {
super(OP_SET_ACL);
}
@@ -4347,7 +4344,7 @@ public abstract class FSEditLogOp {
/**
* Operation corresponding to upgrade
*/
- static class RollingUpgradeOp extends FSEditLogOp { // @Idempotent
+ abstract static class RollingUpgradeOp extends FSEditLogOp { // @Idempotent
private final String name;
private long time;
@@ -4414,7 +4411,7 @@ public abstract class FSEditLogOp {
String path;
byte policyId;
- private SetStoragePolicyOp() {
+ SetStoragePolicyOp() {
super(OP_SET_STORAGE_POLICY);
}
@@ -4480,6 +4477,26 @@ public abstract class FSEditLogOp {
}
}
+ static class RollingUpgradeStartOp extends RollingUpgradeOp {
+ RollingUpgradeStartOp() {
+ super(OP_ROLLING_UPGRADE_START, "start");
+ }
+
+ static RollingUpgradeStartOp getInstance(OpInstanceCache cache) {
+ return (RollingUpgradeStartOp) cache.get(OP_ROLLING_UPGRADE_START);
+ }
+ }
+
+ static class RollingUpgradeFinalizeOp extends RollingUpgradeOp {
+ RollingUpgradeFinalizeOp() {
+ super(OP_ROLLING_UPGRADE_FINALIZE, "finalize");
+ }
+
+ static RollingUpgradeFinalizeOp getInstance(OpInstanceCache cache) {
+ return (RollingUpgradeFinalizeOp) cache.get(OP_ROLLING_UPGRADE_FINALIZE);
+ }
+ }
+
/**
* Class for writing editlog ops
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
index 1a0a296..3f8feba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.*;
/**
* Op codes for edits file
@@ -27,60 +28,64 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public enum FSEditLogOpCodes {
// last op code in file
- OP_ADD ((byte) 0),
- OP_RENAME_OLD ((byte) 1), // deprecated operation
- OP_DELETE ((byte) 2),
- OP_MKDIR ((byte) 3),
- OP_SET_REPLICATION ((byte) 4),
+ OP_ADD ((byte) 0, AddOp.class),
+ // deprecated operation
+ OP_RENAME_OLD ((byte) 1, RenameOldOp.class),
+ OP_DELETE ((byte) 2, DeleteOp.class),
+ OP_MKDIR ((byte) 3, MkdirOp.class),
+ OP_SET_REPLICATION ((byte) 4, SetReplicationOp.class),
@Deprecated OP_DATANODE_ADD ((byte) 5), // obsolete
@Deprecated OP_DATANODE_REMOVE((byte) 6), // obsolete
- OP_SET_PERMISSIONS ((byte) 7),
- OP_SET_OWNER ((byte) 8),
- OP_CLOSE ((byte) 9),
- OP_SET_GENSTAMP_V1 ((byte) 10),
- OP_SET_NS_QUOTA ((byte) 11), // obsolete
- OP_CLEAR_NS_QUOTA ((byte) 12), // obsolete
- OP_TIMES ((byte) 13), // set atime, mtime
- OP_SET_QUOTA ((byte) 14),
- OP_RENAME ((byte) 15), // filecontext rename
- OP_CONCAT_DELETE ((byte) 16), // concat files
- OP_SYMLINK ((byte) 17),
- OP_GET_DELEGATION_TOKEN ((byte) 18),
- OP_RENEW_DELEGATION_TOKEN ((byte) 19),
- OP_CANCEL_DELEGATION_TOKEN ((byte) 20),
- OP_UPDATE_MASTER_KEY ((byte) 21),
- OP_REASSIGN_LEASE ((byte) 22),
- OP_END_LOG_SEGMENT ((byte) 23),
- OP_START_LOG_SEGMENT ((byte) 24),
- OP_UPDATE_BLOCKS ((byte) 25),
- OP_CREATE_SNAPSHOT ((byte) 26),
- OP_DELETE_SNAPSHOT ((byte) 27),
- OP_RENAME_SNAPSHOT ((byte) 28),
- OP_ALLOW_SNAPSHOT ((byte) 29),
- OP_DISALLOW_SNAPSHOT ((byte) 30),
- OP_SET_GENSTAMP_V2 ((byte) 31),
- OP_ALLOCATE_BLOCK_ID ((byte) 32),
- OP_ADD_BLOCK ((byte) 33),
- OP_ADD_CACHE_DIRECTIVE ((byte) 34),
- OP_REMOVE_CACHE_DIRECTIVE ((byte) 35),
- OP_ADD_CACHE_POOL ((byte) 36),
- OP_MODIFY_CACHE_POOL ((byte) 37),
- OP_REMOVE_CACHE_POOL ((byte) 38),
- OP_MODIFY_CACHE_DIRECTIVE ((byte) 39),
- OP_SET_ACL ((byte) 40),
- OP_ROLLING_UPGRADE_START ((byte) 41),
- OP_ROLLING_UPGRADE_FINALIZE ((byte) 42),
- OP_SET_XATTR ((byte) 43),
- OP_REMOVE_XATTR ((byte) 44),
- OP_SET_STORAGE_POLICY ((byte) 45),
- OP_TRUNCATE ((byte) 46),
- OP_APPEND ((byte) 47),
- OP_SET_QUOTA_BY_STORAGETYPE ((byte) 48),
+ OP_SET_PERMISSIONS ((byte) 7, SetPermissionsOp.class),
+ OP_SET_OWNER ((byte) 8, SetOwnerOp.class),
+ OP_CLOSE ((byte) 9, CloseOp.class),
+ OP_SET_GENSTAMP_V1 ((byte) 10, SetGenstampV1Op.class),
+ OP_SET_NS_QUOTA ((byte) 11, SetNSQuotaOp.class), // obsolete
+ OP_CLEAR_NS_QUOTA ((byte) 12, ClearNSQuotaOp.class), // obsolete
+ OP_TIMES ((byte) 13, TimesOp.class), // set atime, mtime
+ OP_SET_QUOTA ((byte) 14, SetQuotaOp.class),
+ // filecontext rename
+ OP_RENAME ((byte) 15, RenameOp.class),
+ // concat files
+ OP_CONCAT_DELETE ((byte) 16, ConcatDeleteOp.class),
+ OP_SYMLINK ((byte) 17, SymlinkOp.class),
+ OP_GET_DELEGATION_TOKEN ((byte) 18, GetDelegationTokenOp.class),
+ OP_RENEW_DELEGATION_TOKEN ((byte) 19, RenewDelegationTokenOp.class),
+ OP_CANCEL_DELEGATION_TOKEN ((byte) 20, CancelDelegationTokenOp.class),
+ OP_UPDATE_MASTER_KEY ((byte) 21, UpdateMasterKeyOp.class),
+ OP_REASSIGN_LEASE ((byte) 22, ReassignLeaseOp.class),
+ OP_END_LOG_SEGMENT ((byte) 23, EndLogSegmentOp.class),
+ OP_START_LOG_SEGMENT ((byte) 24, StartLogSegmentOp.class),
+ OP_UPDATE_BLOCKS ((byte) 25, UpdateBlocksOp.class),
+ OP_CREATE_SNAPSHOT ((byte) 26, CreateSnapshotOp.class),
+ OP_DELETE_SNAPSHOT ((byte) 27, DeleteSnapshotOp.class),
+ OP_RENAME_SNAPSHOT ((byte) 28, RenameSnapshotOp.class),
+ OP_ALLOW_SNAPSHOT ((byte) 29, AllowSnapshotOp.class),
+ OP_DISALLOW_SNAPSHOT ((byte) 30, DisallowSnapshotOp.class),
+ OP_SET_GENSTAMP_V2 ((byte) 31, SetGenstampV2Op.class),
+ OP_ALLOCATE_BLOCK_ID ((byte) 32, AllocateBlockIdOp.class),
+ OP_ADD_BLOCK ((byte) 33, AddBlockOp.class),
+ OP_ADD_CACHE_DIRECTIVE ((byte) 34, AddCacheDirectiveInfoOp.class),
+ OP_REMOVE_CACHE_DIRECTIVE ((byte) 35, RemoveCacheDirectiveInfoOp.class),
+ OP_ADD_CACHE_POOL ((byte) 36, AddCachePoolOp.class),
+ OP_MODIFY_CACHE_POOL ((byte) 37, ModifyCachePoolOp.class),
+ OP_REMOVE_CACHE_POOL ((byte) 38, RemoveCachePoolOp.class),
+ OP_MODIFY_CACHE_DIRECTIVE ((byte) 39, ModifyCacheDirectiveInfoOp.class),
+ OP_SET_ACL ((byte) 40, SetAclOp.class),
+ OP_ROLLING_UPGRADE_START ((byte) 41, RollingUpgradeStartOp.class),
+ OP_ROLLING_UPGRADE_FINALIZE ((byte) 42, RollingUpgradeFinalizeOp.class),
+ OP_SET_XATTR ((byte) 43, SetXAttrOp.class),
+ OP_REMOVE_XATTR ((byte) 44, RemoveXAttrOp.class),
+ OP_SET_STORAGE_POLICY ((byte) 45, SetStoragePolicyOp.class),
+ OP_TRUNCATE ((byte) 46, TruncateOp.class),
+ OP_APPEND ((byte) 47, AppendOp.class),
+ OP_SET_QUOTA_BY_STORAGETYPE ((byte) 48, SetQuotaByStorageTypeOp.class),
// Note that the current range of the valid OP code is 0~127
OP_INVALID ((byte) -1);
private final byte opCode;
+ private final Class<? extends FSEditLogOp> opClass;
/**
* Constructor
@@ -88,7 +93,12 @@ public enum FSEditLogOpCodes {
* @param opCode byte value of constructed enum
*/
FSEditLogOpCodes(byte opCode) {
+ this(opCode, null);
+ }
+
+ FSEditLogOpCodes(byte opCode, Class<? extends FSEditLogOp> opClass) {
this.opCode = opCode;
+ this.opClass = opClass;
}
/**
@@ -100,6 +110,10 @@ public enum FSEditLogOpCodes {
return opCode;
}
+ public Class<? extends FSEditLogOp> getOpClass() {
+ return opClass;
+ }
+
private static final FSEditLogOpCodes[] VALUES;
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 084f82a..b637105 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -140,7 +140,7 @@ public class FSImage implements Closeable {
storage.setRestoreFailedStorage(true);
}
- this.editLog = new FSEditLog(conf, storage, editsDirs);
+ this.editLog = FSEditLog.newInstance(conf, storage, editsDirs);
archivalManager = new NNStorageRetentionManager(conf, storage, editLog);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index f376901..e8900ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1263,7 +1263,6 @@ public class NameNode implements NameNodeStatusMXBean {
newSharedEditLog.logEdit(op);
if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) {
- newSharedEditLog.logSync();
newSharedEditLog.endCurrentLogSegment(false);
LOG.debug("ending log segment because of END_LOG_SEGMENT op in "
+ stream);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
index 54b5c6e..ce0b050 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
@@ -283,8 +283,8 @@ public class NameNodeMetrics {
transactions.add(latency);
}
- public void incrTransactionsBatchedInSync() {
- transactionsBatchedInSync.incr();
+ public void incrTransactionsBatchedInSync(long count) {
+ transactionsBatchedInSync.incr(count);
}
public void addSync(long elapsed) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index dc1853a..0b08996 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -267,6 +267,9 @@ public class DFSTestUtil {
}
public static void setEditLogForTesting(FSNamesystem fsn, FSEditLog newLog) {
+ // spies are shallow copies, must allow async log to restart its thread
+ // so it has the new copy
+ newLog.restart();
Whitebox.setInternalState(fsn.getFSImage(), "editLog", newLog);
Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
index c79e0c2..9b42cac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
@@ -69,17 +69,21 @@ import org.junit.runners.Parameterized.Parameters;
public class TestAuditLogs {
static final String auditLogFile = PathUtils.getTestDirName(TestAuditLogs.class) + "/TestAuditLogs-audit.log";
final boolean useAsyncLog;
-
+ final boolean useAsyncEdits;
+
@Parameters
public static Collection<Object[]> data() {
Collection<Object[]> params = new ArrayList<Object[]>();
- params.add(new Object[]{new Boolean(false)});
- params.add(new Object[]{new Boolean(true)});
+ params.add(new Object[]{Boolean.FALSE, Boolean.FALSE});
+ params.add(new Object[]{Boolean.TRUE, Boolean.FALSE});
+ params.add(new Object[]{Boolean.FALSE, Boolean.TRUE});
+ params.add(new Object[]{Boolean.TRUE, Boolean.TRUE});
return params;
}
-
- public TestAuditLogs(boolean useAsyncLog) {
+
+ public TestAuditLogs(boolean useAsyncLog, boolean useAsyncEdits) {
this.useAsyncLog = useAsyncLog;
+ this.useAsyncEdits = useAsyncEdits;
}
// Pattern for:
@@ -116,6 +120,7 @@ public class TestAuditLogs {
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY, useAsyncLog);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, useAsyncEdits);
util = new DFSTestUtil.Builder().setName("TestAuditAllowed").
setNumFiles(20).build();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 14240e0..1eb377a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -88,6 +88,9 @@ import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.LogManager;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import org.mockito.Mockito;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -98,12 +101,33 @@ import com.google.common.collect.Lists;
/**
* This class tests the creation and validation of a checkpoint.
*/
+@RunWith(Parameterized.class)
public class TestEditLog {
-
+
static {
GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
}
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestEditLog(Boolean async) {
+ useAsyncEditLog = async;
+ }
+
+ public static Configuration getConf() {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
+ return conf;
+ }
+
/**
* A garbage mkdir op which is used for testing
* {@link EditLogFileInputStream#scanEditLog(File)}
@@ -225,11 +249,12 @@ public class TestEditLog {
* @param storage Storage object used by namenode
*/
private static FSEditLog getFSEditLog(NNStorage storage) throws IOException {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
// Make sure the edits dirs are set in the provided configuration object.
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
StringUtils.join(",", storage.getEditsDirectories()));
- FSEditLog log = new FSEditLog(conf, storage, FSNamesystem.getNamespaceEditsDirs(conf));
+ FSEditLog log = FSEditLog.newInstance(
+ conf, storage, FSNamesystem.getNamespaceEditsDirs(conf));
return log;
}
@@ -252,7 +277,7 @@ public class TestEditLog {
*/
@Test
public void testPreTxidEditLogWithEdits() throws Exception {
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
try {
@@ -282,7 +307,7 @@ public class TestEditLog {
@Test
public void testSimpleEditLog() throws IOException {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
try {
@@ -351,7 +376,7 @@ public class TestEditLog {
private void testEditLog(int initialSize) throws IOException {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
@@ -482,8 +507,12 @@ public class TestEditLog {
@Test
public void testSyncBatching() throws Exception {
- // start a cluster
- Configuration conf = new HdfsConfiguration();
+ if (useAsyncEditLog) {
+ // semantics are completely differently since edits will be auto-synced
+ return;
+ }
+ // start a cluster
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
ExecutorService threadA = Executors.newSingleThreadExecutor();
@@ -546,7 +575,7 @@ public class TestEditLog {
@Test
public void testBatchedSyncWithClosedLogs() throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
ExecutorService threadA = Executors.newSingleThreadExecutor();
@@ -586,7 +615,7 @@ public class TestEditLog {
@Test
public void testEditChecksum() throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
@@ -658,7 +687,7 @@ public class TestEditLog {
*/
private void testCrashRecovery(int numTransactions) throws Exception {
MiniDFSCluster cluster = null;
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
CHECKPOINT_ON_STARTUP_MIN_TXNS);
@@ -803,7 +832,7 @@ public class TestEditLog {
boolean updateTransactionIdFile, boolean shouldSucceed)
throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATA_NODES).build();
@@ -1134,7 +1163,7 @@ public class TestEditLog {
public static NNStorage setupEdits(List<URI> editUris, int numrolls,
boolean closeOnFinish, AbortSpec... abortAtRolls) throws IOException {
List<AbortSpec> aborts = new ArrayList<AbortSpec>(Arrays.asList(abortAtRolls));
- NNStorage storage = new NNStorage(new Configuration(),
+ NNStorage storage = new NNStorage(getConf(),
Collections.<URI>emptyList(),
editUris);
storage.format(new NamespaceInfo());
@@ -1296,7 +1325,7 @@ public class TestEditLog {
EditLogFileOutputStream elfos = null;
EditLogFileInputStream elfis = null;
try {
- elfos = new EditLogFileOutputStream(new Configuration(), TEST_LOG_NAME, 0);
+ elfos = new EditLogFileOutputStream(getConf(), TEST_LOG_NAME, 0);
elfos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
elfos.writeRaw(garbage, 0, garbage.length);
elfos.setReadyToFlush();
@@ -1474,7 +1503,7 @@ public class TestEditLog {
public void testManyEditLogSegments() throws IOException {
final int NUM_EDIT_LOG_ROLLS = 1000;
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
index f22ee2f..c60d79f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.net.BindException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Random;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY;
@@ -30,18 +32,40 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NameNodeEditLogRoller;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Supplier;
+@RunWith(Parameterized.class)
public class TestEditLogAutoroll {
+ static {
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestEditLogAutoroll(Boolean async) {
+ useAsyncEditLog = async;
+ }
private Configuration conf;
private MiniDFSCluster cluster;
@@ -61,6 +85,8 @@ public class TestEditLogAutoroll {
// Make it autoroll after 10 edits
conf.setFloat(DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD, 0.5f);
conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS, 100);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
int retryCount = 0;
while (true) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
index 51dfc3e..28169bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
@@ -21,12 +21,13 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -43,13 +44,37 @@ import org.apache.hadoop.util.ExitUtil.ExitException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import org.mockito.Mockito;
+@RunWith(Parameterized.class)
public class TestEditLogJournalFailures {
private int editsPerformed = 0;
private MiniDFSCluster cluster;
private FileSystem fs;
+ private boolean useAsyncEdits;
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{Boolean.FALSE});
+ params.add(new Object[]{Boolean.TRUE});
+ return params;
+ }
+
+ public TestEditLogJournalFailures(boolean useAsyncEdits) {
+ this.useAsyncEdits = useAsyncEdits;
+ }
+
+ private Configuration getConf() {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEdits);
+ return conf;
+ }
/**
* Create the mini cluster for testing and sub in a custom runtime so that
@@ -57,9 +82,9 @@ public class TestEditLogJournalFailures {
*/
@Before
public void setUpMiniCluster() throws IOException {
- setUpMiniCluster(new HdfsConfiguration(), true);
+ setUpMiniCluster(getConf(), true);
}
-
+
public void setUpMiniCluster(Configuration conf, boolean manageNameDfsDirs)
throws IOException {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
@@ -153,7 +178,7 @@ public class TestEditLogJournalFailures {
String[] editsDirs = cluster.getConfiguration(0).getTrimmedStrings(
DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
shutDownMiniCluster();
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, editsDirs[0]);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY, 0);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY, 0);
@@ -193,7 +218,7 @@ public class TestEditLogJournalFailures {
throws IOException {
// Set up 4 name/edits dirs.
shutDownMiniCluster();
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
String[] nameDirs = new String[4];
for (int i = 0; i < nameDirs.length; i++) {
File nameDir = new File(PathUtils.getTestDir(getClass()), "name-dir" + i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
index fcffbc3..195ce5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
@@ -26,14 +26,17 @@ import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -46,10 +49,14 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -57,15 +64,27 @@ import org.mockito.stubbing.Answer;
* This class tests various synchronization bugs in FSEditLog rolling
* and namespace saving.
*/
+@RunWith(Parameterized.class)
public class TestEditLogRace {
static {
GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
}
- private static final Log LOG = LogFactory.getLog(TestEditLogRace.class);
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ false });
+ params.add(new Object[]{ true });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
- private static final String NAME_DIR =
- MiniDFSCluster.getBaseDirectory() + "name1";
+ public TestEditLogRace(boolean useAsyncEditLog) {
+ TestEditLogRace.useAsyncEditLog = useAsyncEditLog;
+ }
+
+ private static final Log LOG = LogFactory.getLog(TestEditLogRace.class);
// This test creates NUM_THREADS threads and each thread continuously writes
// transactions
@@ -94,21 +113,29 @@ public class TestEditLogRace {
* This value needs to be significantly longer than the average
* time for an fsync() or enterSafeMode().
*/
- private static final int BLOCK_TIME = 10;
-
+ private static final int BLOCK_TIME = 4; // 4 sec pretty generous
+
//
// an object that does a bunch of transactions
//
static class Transactions implements Runnable {
final NamenodeProtocols nn;
+ final MiniDFSCluster cluster;
+ FileSystem fs;
short replication = 3;
long blockSize = 64;
volatile boolean stopped = false;
volatile Thread thr;
final AtomicReference<Throwable> caught;
- Transactions(NamenodeProtocols ns, AtomicReference<Throwable> caught) {
- nn = ns;
+ Transactions(MiniDFSCluster cluster, AtomicReference<Throwable> caught) {
+ this.cluster = cluster;
+ this.nn = cluster.getNameNodeRpc();
+ try {
+ this.fs = cluster.getFileSystem();
+ } catch (IOException e) {
+ caught.set(e);
+ }
this.caught = caught;
}
@@ -122,11 +149,23 @@ public class TestEditLogRace {
while (!stopped) {
try {
String dirname = "/thr-" + thr.getId() + "-dir-" + i;
- nn.mkdirs(dirname, p, true);
- nn.delete(dirname, true);
+ if (i % 2 == 0) {
+ Path dirnamePath = new Path(dirname);
+ fs.mkdirs(dirnamePath);
+ fs.delete(dirnamePath, true);
+ } else {
+ nn.mkdirs(dirname, p, true);
+ nn.delete(dirname, true);
+ }
} catch (SafeModeException sme) {
// This is OK - the tests will bring NN in and out of safemode
} catch (Throwable e) {
+ // This is OK - the tests will bring NN in and out of safemode
+ if (e instanceof RemoteException &&
+ ((RemoteException)e).getClassName()
+ .contains("SafeModeException")) {
+ return;
+ }
LOG.warn("Got error in transaction thread", e);
caught.compareAndSet(null, e);
break;
@@ -144,11 +183,11 @@ public class TestEditLogRace {
}
}
- private void startTransactionWorkers(NamenodeProtocols namesystem,
+ private void startTransactionWorkers(MiniDFSCluster cluster,
AtomicReference<Throwable> caughtErr) {
// Create threads and make them run transactions concurrently.
for (int i = 0; i < NUM_THREADS; i++) {
- Transactions trans = new Transactions(namesystem, caughtErr);
+ Transactions trans = new Transactions(cluster, caughtErr);
new Thread(trans, "TransactionThread-" + i).start();
workers.add(trans);
}
@@ -174,21 +213,21 @@ public class TestEditLogRace {
@Test
public void testEditLogRolling() throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = null;
+ Configuration conf = getConf();
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
FileSystem fileSys = null;
AtomicReference<Throwable> caughtErr = new AtomicReference<Throwable>();
try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
final NamenodeProtocols nn = cluster.getNameNode().getRpcServer();
FSImage fsimage = cluster.getNamesystem().getFSImage();
StorageDirectory sd = fsimage.getStorage().getStorageDir(0);
- startTransactionWorkers(nn, caughtErr);
+ startTransactionWorkers(cluster, caughtErr);
long previousLogTxId = 1;
@@ -256,7 +295,7 @@ public class TestEditLogRace {
@Test
public void testSaveNamespace() throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
@@ -266,12 +305,11 @@ public class TestEditLogRace {
cluster.waitActive();
fileSys = cluster.getFileSystem();
final FSNamesystem namesystem = cluster.getNamesystem();
- final NamenodeProtocols nn = cluster.getNameNodeRpc();
FSImage fsimage = namesystem.getFSImage();
FSEditLog editLog = fsimage.getEditLog();
- startTransactionWorkers(nn, caughtErr);
+ startTransactionWorkers(cluster, caughtErr);
for (int i = 0; i < NUM_SAVE_IMAGE && caughtErr.get() == null; i++) {
try {
@@ -321,11 +359,13 @@ public class TestEditLogRace {
private Configuration getConf() {
Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, NAME_DIR);
- conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, NAME_DIR);
- conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
+ //conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, NAME_DIR);
+ //conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, NAME_DIR);
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
return conf;
}
@@ -389,7 +429,7 @@ public class TestEditLogRace {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
LOG.info("Flush called");
- if (Thread.currentThread() == doAnEditThread) {
+ if (useAsyncEditLog || Thread.currentThread() == doAnEditThread) {
LOG.info("edit thread: Telling main thread we made it to flush section...");
// Signal to main thread that the edit thread is in the racy section
waitToEnterFlush.countDown();
@@ -457,62 +497,52 @@ public class TestEditLogRace {
try {
FSImage fsimage = namesystem.getFSImage();
- FSEditLog editLog = spy(fsimage.getEditLog());
- DFSTestUtil.setEditLogForTesting(namesystem, editLog);
+ final FSEditLog editLog = fsimage.getEditLog();
final AtomicReference<Throwable> deferredException =
new AtomicReference<Throwable>();
- final CountDownLatch waitToEnterSync = new CountDownLatch(1);
-
+ final CountDownLatch sleepingBeforeSync = new CountDownLatch(1);
+
final Thread doAnEditThread = new Thread() {
@Override
public void run() {
try {
- LOG.info("Starting mkdirs");
- namesystem.mkdirs("/test",
- new PermissionStatus("test","test", new FsPermission((short)00755)),
- true);
- LOG.info("mkdirs complete");
+ LOG.info("Starting setOwner");
+ namesystem.writeLock();
+ try {
+ editLog.logSetOwner("/","test","test");
+ } finally {
+ namesystem.writeUnlock();
+ }
+ sleepingBeforeSync.countDown();
+ LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
+ Thread.sleep(BLOCK_TIME*1000);
+ editLog.logSync();
+ LOG.info("edit thread: logSync complete");
} catch (Throwable ioe) {
LOG.fatal("Got exception", ioe);
deferredException.set(ioe);
- waitToEnterSync.countDown();
- }
- }
- };
-
- Answer<Void> blockingSync = new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- LOG.info("logSync called");
- if (Thread.currentThread() == doAnEditThread) {
- LOG.info("edit thread: Telling main thread we made it just before logSync...");
- waitToEnterSync.countDown();
- LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
- Thread.sleep(BLOCK_TIME*1000);
- LOG.info("Going through to logSync. This will allow the main thread to continue.");
+ sleepingBeforeSync.countDown();
}
- invocation.callRealMethod();
- LOG.info("logSync complete");
- return null;
}
};
- doAnswer(blockingSync).when(editLog).logSync();
-
+ doAnEditThread.setDaemon(true);
doAnEditThread.start();
LOG.info("Main thread: waiting to just before logSync...");
- waitToEnterSync.await();
+ sleepingBeforeSync.await(200, TimeUnit.MILLISECONDS);
assertNull(deferredException.get());
LOG.info("Main thread: detected that logSync about to be called.");
LOG.info("Trying to enter safe mode.");
- LOG.info("This should block for " + BLOCK_TIME + "sec, since we have pending edits");
-
+
long st = Time.now();
namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
long et = Time.now();
- LOG.info("Entered safe mode");
- // Make sure we really waited for the flush to complete!
- assertTrue(et - st > (BLOCK_TIME - 1)*1000);
+ LOG.info("Entered safe mode after "+(et-st)+"ms");
+
+ // Make sure we didn't wait for the thread that did a logEdit but
+ // not logSync. Going into safemode does a logSyncAll that will flush
+ // its edit.
+ assertTrue(et - st < (BLOCK_TIME/2)*1000);
// Once we're in safe mode, save namespace.
namesystem.saveNamespace(0, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21517168/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index b0e5704..fe29e1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -31,6 +31,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
@@ -59,28 +61,51 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
+@RunWith(Parameterized.class)
public class TestFSEditLogLoader {
-
+ @Parameters
+ public static Collection<Object[]> data() {
+ Collection<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[]{ Boolean.FALSE });
+ params.add(new Object[]{ Boolean.TRUE });
+ return params;
+ }
+
+ private static boolean useAsyncEditLog;
+ public TestFSEditLogLoader(Boolean async) {
+ useAsyncEditLog = async;
+ }
+
+ private static Configuration getConf() {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLog);
+ return conf;
+ }
+
static {
GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
GenericTestUtils.setLogLevel(FSEditLogLoader.LOG, Level.ALL);
}
-
+
private static final File TEST_DIR = PathUtils.getTestDir(TestFSEditLogLoader.class);
private static final int NUM_DATA_NODES = 0;
private static final ErasureCodingPolicy testECPolicy
= StripedFileTestUtil.TEST_EC_POLICY;
-
+
@Test
public void testDisplayRecentEditLogOpCodes() throws IOException {
- // start a cluster
- Configuration conf = new HdfsConfiguration();
+ // start a cluster
+ Configuration conf = getConf();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
@@ -130,7 +155,7 @@ public class TestFSEditLogLoader {
@Test
public void testReplicationAdjusted() throws Exception {
// start a cluster
- Configuration conf = new HdfsConfiguration();
+ Configuration conf = getConf();
// Replicate and heartbeat fast to shave a few seconds off test
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
[43/50] [abbrv] hadoop git commit: HADOOP-12793. Write a new group
mapping service guide (Wei-Chiu Chuang via iwasakims)
Posted by ar...@apache.org.
HADOOP-12793. Write a new group mapping service guide (Wei-Chiu Chuang via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f70e317c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f70e317c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f70e317c
Branch: refs/heads/HDFS-1312
Commit: f70e317cccf14185cc24ea3cb57ffc63b00158cc
Parents: 27941a1
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Thu Mar 3 17:14:32 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Thu Mar 3 17:14:32 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../src/main/resources/core-default.xml | 114 +++--------
.../src/site/markdown/GroupsMapping.md | 191 +++++++++++++++++++
.../src/site/markdown/SecureMode.md | 2 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 8 +-
5 files changed, 218 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f70e317c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f8e033d..dc93beb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1173,6 +1173,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12622. Improve the loggings in RetryPolicies and RetryInvocationHandler.
(Junping Du via jianhe)
+ HADOOP-12793. Write a new group mapping service guide
+ (Wei-Chiu Chuang via iwasakims)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f70e317c/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 57f5274..187f923 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -131,98 +131,6 @@
</description>
</property>
-<!--
-=== Multiple group mapping providers configuration sample ===
- This sample illustrates a typical use case for CompositeGroupsMapping where
-Hadoop authentication uses MIT Kerberos which trusts an AD realm. In this case, service
-principals such as hdfs, mapred, hbase, hive, oozie and etc can be placed in In MIT Kerberos,
-but end users are just from the trusted AD. For the service principals, ShellBasedUnixGroupsMapping
-provider can be used to query their groups for efficiency, and for end users, LdapGroupsMapping
-provider can be used. This avoids to add group entries in AD for service principals when only using
-LdapGroupsMapping provider.
- In case multiple ADs are involved and trusted by the MIT Kerberos in this use case, LdapGroupsMapping
-provider can be used more times with different AD specific configurations. This sample also shows how
-to do that. Here are the necessary configurations.
-
-<property>
- <name>hadoop.security.group.mapping</name>
- <value>org.apache.hadoop.security.CompositeGroupsMapping</value>
- <description>
- Class for user to group mapping (get groups for a given user) for ACL, which
- makes use of other multiple providers to provide the service.
- </description>
-</property>
-
-<property>
- <name>hadoop.security.group.mapping.providers</name>
- <value>shell4services,ad4usersX,ad4usersY</value>
- <description>
- Comma separated of names of other providers to provide user to group mapping.
- </description>
-</property>
-
-<property>
- <name>hadoop.security.group.mapping.providers.combined</name>
- <value>true</value>
- <description>
- true or false to indicate whether groups from the providers are combined or not. The default value is true
- If true, then all the providers will be tried to get groups and all the groups are combined to return as
- the final results. Otherwise, providers are tried one by one in the configured list order, and if any
- groups are retrieved from any provider, then the groups will be returned without trying the left ones.
- </description>
-</property>
-
-<property>
- <name>hadoop.security.group.mapping.provider.shell4services</name>
- <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
- <description>
- Class for group mapping provider named by 'shell4services'. The name can then be referenced
- by hadoop.security.group.mapping.providers property.
- </description>
-</property>
-
-<property>
- <name>hadoop.security.group.mapping.provider.ad4usersX</name>
- <value>org.apache.hadoop.security.LdapGroupsMapping</value>
- <description>
- Class for group mapping provider named by 'ad4usersX'. The name can then be referenced
- by hadoop.security.group.mapping.providers property.
- </description>
-</property>
-
-<property>
- <name>hadoop.security.group.mapping.provider.ad4usersY</name>
- <value>org.apache.hadoop.security.LdapGroupsMapping</value>
- <description>
- Class for group mapping provider named by 'ad4usersY'. The name can then be referenced
- by hadoop.security.group.mapping.providers property.
- </description>
-</property>
-
-<property>
-<name>hadoop.security.group.mapping.provider.ad4usersX.ldap.url</name>
-<value>ldap://ad-host-for-users-X:389</value>
- <description>
- ldap url for the provider named by 'ad4usersX'. Note this property comes from
- 'hadoop.security.group.mapping.ldap.url'.
- </description>
-</property>
-
-<property>
-<name>hadoop.security.group.mapping.provider.ad4usersY.ldap.url</name>
-<value>ldap://ad-host-for-users-Y:389</value>
- <description>
- ldap url for the provider named by 'ad4usersY'. Note this property comes from
- 'hadoop.security.group.mapping.ldap.url'.
- </description>
-</property>
-
-You also need to configure other properties like
- hadoop.security.group.mapping.ldap.bind.password.file and etc.
-for ldap providers in the same way as above does.
-
--->
-
<property>
<name>hadoop.security.groups.cache.secs</name>
<value>300</value>
@@ -401,6 +309,28 @@ for ldap providers in the same way as above does.
</property>
<property>
+ <name>hadoop.security.group.mapping.providers</name>
+ <value></value>
+ <description>
+ Comma separated of names of other providers to provide user to group
+ mapping. Used by CompositeGroupsMapping.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.providers.combined</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether groups from the providers are combined or
+ not. The default value is true. If true, then all the providers will be
+ tried to get groups and all the groups are combined to return as the final
+ results. Otherwise, providers are tried one by one in the configured list
+ order, and if any groups are retrieved from any provider, then the groups
+ will be returned without trying the left ones.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.service.user.name.key</name>
<value></value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f70e317c/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
new file mode 100644
index 0000000..5a67bd1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -0,0 +1,191 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Groups Mapping
+===================
+
+* [Hadoop Groups Mapping](#Hadoop_Groups_Mapping)
+ * [Overview](#Overview)
+ * [LDAP Groups Mapping](#LDAP_Groups_Mapping)
+ * [Active Directory](#Active_Directory)
+ * [POSIX Groups](#POSIX_Groups)
+ * [SSL](#SSL)
+ * [Composite Groups Mapping](#Composite_Groups_Mapping)
+ * [Multiple group mapping providers configuration sample](#Multiple_group_mapping_providers_configuration_sample)
+
+Overview
+--------
+The groups of a user is determined by a group mapping service provider.
+Hadoop supports various group mapping mechanisms, configured by the `hadoop.security.group.mapping` property. Some of them, such as `JniBasedUnixGroupsMappingWithFallback`, use operating
+systems' group name resolution and requires no configuration. But Hadoop also supports special group mapping mechanisms through
+LDAP and composition of LDAP and operating system group name resolution, which require additional configurations.
+`hadoop.security.group.mapping` can be one of the following:
+
+* **org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback**
+
+ The default implementation. It will determine if the Java Native Interface (JNI) is available. If JNI is available, the implementation will use the API within hadoop to resolve a list of groups for a user. If JNI is not available then the shell-based implementation, `ShellBasedUnixGroupsMapping`, is used.
+
+* **org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMappingWithFallback**
+
+ Similar to `JniBasedUnixGroupsMappingWithFallback`. If JNI is available, it obtains netgroup membership using the Hadoop native API; otherwise uses `ShellBasedUnixGroupsNetgroupMapping`.
+
+* **org.apache.hadoop.security.ShellBasedUnixGroupsMapping**
+
+ This implementation shells out with the `bash -c groups` command (for a Linux/Unix environment) or the `net group` command (for a Windows environment) to resolve a list of groups for a user.
+
+* **org.apache.hadoop.security.ShellBasedUnixGroupsNetgroupMapping**
+
+ This implementation is similar to `ShellBasedUnixGroupsMapping`, except that it executes `getent netgroup` command to get netgroup membership.
+
+* **org.apache.hadoop.security.LdapGroupsMapping**
+
+ An alternate implementation, which connects directly to an LDAP server to resolve the list of groups. However, this provider should only be used if the required groups reside exclusively in LDAP, and are not materialized on the Unix servers.
+ LdapGroupsMapping supports SSL connection and POSIX group semantics. See section [LDAP Groups Mapping](#LDAP_Groups_Mapping) for details.
+
+* **org.apache.hadoop.security.CompositeGroupsMapping**
+
+ This implementation composites other group mapping providers for determining group membership. This allows to combine existing provider implementations and composite a virtually new provider without customized development to deal with complex situation. See section [Composite Groups Mapping](#Composite_Groups_Mapping) for details.
+
+For HDFS, the mapping of users to groups is performed on the NameNode. Thus, the host system configuration of the NameNode determines the group mappings for the users.
+
+Note that HDFS stores the user and group of a file or directory as strings; there is no conversion from user and group identity numbers as is conventional in Unix.
+
+
+LDAP Groups Mapping
+--------
+This provider supports LDAP with simple password authentication using JNDI API.
+`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server for resolving user groups.
+
+`hadoop.security.group.mapping.ldap.base` configures the search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.
+
+If the LDAP server does not support anonymous binds,
+set the distinguished name of the user to bind in `hadoop.security.group.mapping.ldap.bind.user`.
+The path to the file containing the bind user's password is specified in `hadoop.security.group.mapping.ldap.bind.password.file`.
+This file should be readable only by the Unix user running the daemons.
+
+It is possible to set a maximum time limit when searching and awaiting a result.
+Set `hadoop.security.group.mapping.ldap.directory.search.timeout` to 0 if infinite wait period is desired. Default is 10,000 milliseconds (10 seconds).
+
+The implementation does not attempt to resolve group hierarchies. Therefore, a user must be an explicit member of a group object
+in order to be considered a member.
+
+
+### Active Directory ###
+The default configuration supports LDAP group name resolution with an Active Directory server.
+
+### POSIX Groups ###
+If the LDAP server supports POSIX group semantics, Hadoop can perform LDAP group resolution queries to the server by setting both
+`hadoop.security.group.mapping.ldap.search.filter.user` to `posixAccount` and
+`hadoop.security.group.mapping.ldap.search.filter.group` to `posixGroup`.
+
+### SSL ###
+To secure the connection, the implementation supports LDAP over SSL (LDAPS). SSL is enable by setting `hadoop.security.group.mapping.ldap.ssl` to `true`.
+In addition, specify the path to the keystore file for SSL connection in `hadoop.security.group.mapping.ldap.ssl.keystore` and keystore password in `hadoop.security.group.mapping.ldap.ssl.keystore.password`.
+Alternatively, store the keystore password in a file, and point `hadoop.security.group.mapping.ldap.ssl.keystore.password.file` to that file. For security purposes, this file should be readable only by the Unix user running the daemons.
+
+Composite Groups Mapping
+--------
+`CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.
+It get groups from each of the providers in the list one after the other. If `hadoop.security.group.mapping.providers.combined` is `true`, merge the groups returned by all providers; otherwise, return the groups in the first successful provider.
+See the following section for a sample configuration.
+
+### Multiple group mapping providers configuration sample ###
+ This sample illustrates a typical use case for `CompositeGroupsMapping` where
+Hadoop authentication uses MIT Kerberos which trusts an AD realm. In this case, service
+principals such as hdfs, mapred, hbase, hive, oozie and etc can be placed in MIT Kerberos,
+but end users are just from the trusted AD. For the service principals, `ShellBasedUnixGroupsMapping`
+provider can be used to query their groups for efficiency, and for end users, `LdapGroupsMapping`
+provider can be used. This avoids to add group entries in AD for service principals when only using
+`LdapGroupsMapping` provider.
+ In case multiple ADs are involved and trusted by the MIT Kerberos, `LdapGroupsMapping`
+provider can be used multiple times with different AD specific configurations. This sample also shows how
+to do that. Here are the necessary configurations.
+
+```<property>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.CompositeGroupsMapping</value>
+ <description>
+ Class for user to group mapping (get groups for a given user) for ACL, which
+ makes use of other multiple providers to provide the service.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.providers</name>
+ <value>shell4services,ad4usersX,ad4usersY</value>
+ <description>
+ Comma separated of names of other providers to provide user to group mapping.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.providers.combined</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether groups from the providers are combined or not. The default value is true
+ If true, then all the providers will be tried to get groups and all the groups are combined to return as
+ the final results. Otherwise, providers are tried one by one in the configured list order, and if any
+ groups are retrieved from any provider, then the groups will be returned without trying the left ones.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.shell4services</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'shell4services'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.ad4usersX</name>
+ <value>org.apache.hadoop.security.LdapGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'ad4usersX'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.ad4usersY</name>
+ <value>org.apache.hadoop.security.LdapGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'ad4usersY'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+<name>hadoop.security.group.mapping.provider.ad4usersX.ldap.url</name>
+<value>ldap://ad-host-for-users-X:389</value>
+ <description>
+ ldap url for the provider named by 'ad4usersX'. Note this property comes from
+ 'hadoop.security.group.mapping.ldap.url'.
+ </description>
+</property>
+
+<property>
+<name>hadoop.security.group.mapping.provider.ad4usersY.ldap.url</name>
+<value>ldap://ad-host-for-users-Y:389</value>
+ <description>
+ ldap url for the provider named by 'ad4usersY'. Note this property comes from
+ 'hadoop.security.group.mapping.ldap.url'.
+ </description>
+</property>
+```
+
+You also need to configure other properties like
+ hadoop.security.group.mapping.ldap.bind.password.file and etc.
+for ldap providers in the same way as above does.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f70e317c/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index 6bfe5fe..9607263 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -184,7 +184,7 @@ Custom rules can be tested using the `hadoop kerbname` command. This command al
### Mapping from user to group
-The system user to system group mapping mechanism can be configured via `hadoop.security.group.mapping`. See [HDFS Permissions Guide](../hadoop-hdfs/HdfsPermissionsGuide.html#Group_Mapping) for details.
+The system user to system group mapping mechanism can be configured via `hadoop.security.group.mapping`. See [Hadoop Groups Mapping](GroupsMapping.html) for details.
Practically you need to manage SSO environment using Kerberos with LDAP for Hadoop in secure mode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f70e317c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index b6b4c6a..58b64a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -74,13 +74,7 @@ Regardless of the mode of operation, the user identity mechanism is extrinsic to
Group Mapping
-------------
-Once a username has been determined as described above, the list of groups is determined by a group mapping service, configured by the hadoop.security.group.mapping property. The default implementation, org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback, will determine if the Java Native Interface (JNI) is available. If JNI is available, the implementation will use the API within hadoop to resolve a list of groups for a user. If JNI is not available then the shell implementation, org.apache.hadoop.security.ShellBasedUnixGroupsMapping, is used. This implementation shells out with the `bash -c groups` command (for a Linux/Unix environment) or the `net group` command (for a Windows environment) to resolve a list of groups for a user.
-
-An alternate implementation, which connects directly to an LDAP server to resolve the list of groups, is available via org.apache.hadoop.security.LdapGroupsMapping. However, this provider should only be used if the required groups reside exclusively in LDAP, and are not materialized on the Unix servers. More information on configuring the group mapping service is available in the Javadocs.
-
-For HDFS, the mapping of users to groups is performed on the NameNode. Thus, the host system configuration of the NameNode determines the group mappings for the users.
-
-Note that HDFS stores the user and group of a file or directory as strings; there is no conversion from user and group identity numbers as is conventional in Unix.
+Once a username has been determined as described above, the list of groups is determined by a group mapping service, configured by the `hadoop.security.group.mapping` property. See [Hadoop Groups Mapping](../hadoop-common/GroupsMapping.html) for details.
Permission Checks
-----------------
[09/50] [abbrv] hadoop git commit: HDFS-9866.
BlockManager#chooseExcessReplicasStriped may weaken rack fault tolerance.
Contributed by Jing Zhao.
Posted by ar...@apache.org.
HDFS-9866. BlockManager#chooseExcessReplicasStriped may weaken rack fault tolerance. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/408f2c80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/408f2c80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/408f2c80
Branch: refs/heads/HDFS-1312
Commit: 408f2c807bbaaaa37ce1b69a5dfa9d76ed427d6e
Parents: 7634d40
Author: Jing Zhao <ji...@apache.org>
Authored: Sun Feb 28 14:54:49 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Sun Feb 28 14:54:49 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 6 +-
.../blockmanagement/BlockPlacementPolicy.java | 48 ++++---
.../BlockPlacementPolicyDefault.java | 20 +--
.../blockmanagement/ErasureCodingWork.java | 6 +
...constructStripedBlocksWithRackAwareness.java | 133 ++++++++++---------
.../blockmanagement/TestReplicationPolicy.java | 15 ++-
.../TestReplicationPolicyWithNodeGroup.java | 2 +-
.../TestReplicationPolicyWithUpgradeDomain.java | 12 +-
9 files changed, 133 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 81b48c3..651f0a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -446,6 +446,9 @@ Trunk (Unreleased)
HDFS-9734. Refactoring of checksum failure report related codes.
(Kai Zheng via zhz)
+ HDFS-9866. BlockManager#chooseExcessReplicasStriped may weaken rack fault
+ tolerance. (jing9)
+
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index f483d8e..77eea0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3245,7 +3245,7 @@ public class BlockManager implements BlockStatsMXBean {
DatanodeDescriptor delNodeHint, List<StorageType> excessTypes) {
BlockPlacementPolicy replicator = placementPolicies.getPolicy(false);
List<DatanodeStorageInfo> replicasToDelete = replicator
- .chooseReplicasToDelete(nonExcess, replication, excessTypes,
+ .chooseReplicasToDelete(nonExcess, nonExcess, replication, excessTypes,
addedNode, delNodeHint);
for (DatanodeStorageInfo choosenReplica : replicasToDelete) {
processChosenExcessReplica(nonExcess, choosenReplica, storedBlock);
@@ -3316,8 +3316,8 @@ public class BlockManager implements BlockStatsMXBean {
internalBlock.setBlockId(storedBlock.getBlockId() + targetIndex);
while (candidates.size() > 1) {
List<DatanodeStorageInfo> replicasToDelete = placementPolicy
- .chooseReplicasToDelete(candidates, (short) 1, excessTypes, null,
- null);
+ .chooseReplicasToDelete(nonExcess, candidates, (short) 1,
+ excessTypes, null, null);
for (DatanodeStorageInfo chosen : replicasToDelete) {
processChosenExcessReplica(nonExcess, chosen, storedBlock);
candidates.remove(chosen);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 8478387..92b03d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -77,8 +77,6 @@ public abstract class BlockPlacementPolicy {
BlockStoragePolicy storagePolicy);
/**
- * Same as {@link #chooseTarget(String, int, Node, Set, long, List, StorageType)}
- * with added parameter {@code favoredDatanodes}
* @param favoredNodes datanodes that should be favored as targets. This
* is only a hint and due to cluster state, namenode may not be
* able to place the blocks on these datanodes.
@@ -106,17 +104,21 @@ public abstract class BlockPlacementPolicy {
* @param numOfReplicas replica number of file to be verified
* @return the result of verification
*/
- abstract public BlockPlacementStatus verifyBlockPlacement(
+ public abstract BlockPlacementStatus verifyBlockPlacement(
DatanodeInfo[] locs, int numOfReplicas);
/**
* Select the excess replica storages for deletion based on either
* delNodehint/Excess storage types.
*
- * @param candidates
+ * @param availableReplicas
* available replicas
+ * @param delCandidates
+ * Candidates for deletion. For normal replication, this set is the
+ * same with availableReplicas. For striped blocks, this set is a
+ * subset of availableReplicas.
* @param expectedNumOfReplicas
- * The required number of replicas for this block
+ * The expected number of replicas remaining in the delCandidates
* @param excessTypes
* type of the storagepolicy
* @param addedNode
@@ -125,10 +127,12 @@ public abstract class BlockPlacementPolicy {
* Hint for excess storage selection
* @return Returns the list of excess replicas chosen for deletion
*/
- abstract public List<DatanodeStorageInfo> chooseReplicasToDelete(
- Collection<DatanodeStorageInfo> candidates, int expectedNumOfReplicas,
+ public abstract List<DatanodeStorageInfo> chooseReplicasToDelete(
+ Collection<DatanodeStorageInfo> availableReplicas,
+ Collection<DatanodeStorageInfo> delCandidates, int expectedNumOfReplicas,
List<StorageType> excessTypes, DatanodeDescriptor addedNode,
DatanodeDescriptor delNodeHint);
+
/**
* Used to setup a BlockPlacementPolicy object. This should be defined by
* all implementations of a BlockPlacementPolicy.
@@ -137,7 +141,7 @@ public abstract class BlockPlacementPolicy {
* @param stats retrieve cluster status from here
* @param clusterMap cluster topology
*/
- abstract protected void initialize(Configuration conf, FSClusterStats stats,
+ protected abstract void initialize(Configuration conf, FSClusterStats stats,
NetworkTopology clusterMap,
Host2NodesMap host2datanodeMap);
@@ -149,7 +153,7 @@ public abstract class BlockPlacementPolicy {
* @param source source replica of the move
* @param target target replica of the move
*/
- abstract public boolean isMovable(Collection<DatanodeInfo> candidates,
+ public abstract boolean isMovable(Collection<DatanodeInfo> candidates,
DatanodeInfo source, DatanodeInfo target);
/**
@@ -191,10 +195,8 @@ public abstract class BlockPlacementPolicy {
"class " + datanode.getClass().getName() + " not allowed");
if (datanode instanceof DatanodeInfo) {
return ((DatanodeInfo)datanode);
- } else if (datanode instanceof DatanodeStorageInfo) {
- return ((DatanodeStorageInfo)datanode).getDatanodeDescriptor();
} else {
- return null;
+ return ((DatanodeStorageInfo)datanode).getDatanodeDescriptor();
}
}
@@ -209,35 +211,37 @@ public abstract class BlockPlacementPolicy {
/**
* Split data nodes into two sets, one set includes nodes on rack with
* more than one replica, the other set contains the remaining nodes.
- *
- * @param storagesOrDataNodes DatanodeStorageInfo/DatanodeInfo to be split
+ *
+ * @param availableSet all the available DataNodes/storages of the block
+ * @param candidates DatanodeStorageInfo/DatanodeInfo to be split
* into two sets
* @param rackMap a map from rack to datanodes
* @param moreThanOne contains nodes on rack with more than one replica
* @param exactlyOne remains contains the remaining nodes
*/
public <T> void splitNodesWithRack(
- final Iterable<T> storagesOrDataNodes,
+ final Iterable<T> availableSet,
+ final Collection<T> candidates,
final Map<String, List<T>> rackMap,
final List<T> moreThanOne,
final List<T> exactlyOne) {
- for(T s: storagesOrDataNodes) {
+ for(T s: availableSet) {
final String rackName = getRack(getDatanodeInfo(s));
List<T> storageList = rackMap.get(rackName);
if (storageList == null) {
- storageList = new ArrayList<T>();
+ storageList = new ArrayList<>();
rackMap.put(rackName, storageList);
}
storageList.add(s);
}
- // split nodes into two sets
- for(List<T> storageList : rackMap.values()) {
- if (storageList.size() == 1) {
+ for (T candidate : candidates) {
+ final String rackName = getRack(getDatanodeInfo(candidate));
+ if (rackMap.get(rackName).size() == 1) {
// exactlyOne contains nodes on rack with only one replica
- exactlyOne.add(storageList.get(0));
+ exactlyOne.add(candidate);
} else {
// moreThanOne contains nodes on rack with more than one replica
- moreThanOne.addAll(storageList);
+ moreThanOne.add(candidate);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index e1a47ae..f20f5fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -972,7 +972,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
@Override
public List<DatanodeStorageInfo> chooseReplicasToDelete(
- Collection<DatanodeStorageInfo> candidates,
+ Collection<DatanodeStorageInfo> availableReplicas,
+ Collection<DatanodeStorageInfo> delCandidates,
int expectedNumOfReplicas,
List<StorageType> excessTypes,
DatanodeDescriptor addedNode,
@@ -985,28 +986,29 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
final List<DatanodeStorageInfo> moreThanOne = new ArrayList<>();
final List<DatanodeStorageInfo> exactlyOne = new ArrayList<>();
- // split nodes into two sets
+ // split candidate nodes for deletion into two sets
// moreThanOne contains nodes on rack with more than one replica
// exactlyOne contains the remaining nodes
- splitNodesWithRack(candidates, rackMap, moreThanOne, exactlyOne);
+ splitNodesWithRack(availableReplicas, delCandidates, rackMap, moreThanOne,
+ exactlyOne);
// pick one node to delete that favors the delete hint
// otherwise pick one with least space from priSet if it is not empty
// otherwise one node with least space from remains
boolean firstOne = true;
final DatanodeStorageInfo delNodeHintStorage =
- DatanodeStorageInfo.getDatanodeStorageInfo(candidates, delNodeHint);
+ DatanodeStorageInfo.getDatanodeStorageInfo(delCandidates, delNodeHint);
final DatanodeStorageInfo addedNodeStorage =
- DatanodeStorageInfo.getDatanodeStorageInfo(candidates, addedNode);
+ DatanodeStorageInfo.getDatanodeStorageInfo(delCandidates, addedNode);
- while (candidates.size() - expectedNumOfReplicas > excessReplicas.size()) {
+ while (delCandidates.size() - expectedNumOfReplicas > excessReplicas.size()) {
final DatanodeStorageInfo cur;
if (firstOne && useDelHint(delNodeHintStorage, addedNodeStorage,
moreThanOne, exactlyOne, excessTypes)) {
cur = delNodeHintStorage;
} else { // regular excessive replica removal
- cur = chooseReplicaToDelete(moreThanOne, exactlyOne, excessTypes,
- rackMap);
+ cur = chooseReplicaToDelete(moreThanOne, exactlyOne,
+ excessTypes, rackMap);
}
firstOne = false;
if (cur == null) {
@@ -1056,7 +1058,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
final Map<String, List<DatanodeInfo>> rackMap = new HashMap<>();
final List<DatanodeInfo> moreThanOne = new ArrayList<>();
final List<DatanodeInfo> exactlyOne = new ArrayList<>();
- splitNodesWithRack(locs, rackMap, moreThanOne, exactlyOne);
+ splitNodesWithRack(locs, locs, rackMap, moreThanOne, exactlyOne);
return notReduceNumOfGroups(moreThanOne, source, target);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
index 38ad324..7877c56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.Node;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
@@ -137,6 +138,11 @@ class ErasureCodingWork extends BlockReconstructionWork {
stripedBlk.getBlockId() + blockIndex, internBlkLen,
stripedBlk.getGenerationStamp());
source.addBlockToBeReplicated(targetBlk, getTargets());
+ if (BlockManager.LOG.isDebugEnabled()) {
+ BlockManager.LOG.debug("Add replication task from source {} to " +
+ "targets {} for EC block {}", source, Arrays.toString(getTargets()),
+ targetBlk);
+ }
} else {
getTargets()[0].getDatanodeDescriptor().addBlockToBeErasureCoded(
new ExtendedBlock(blockPoolId, stripedBlk),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
index 2164957..d269a9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -23,8 +23,9 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -39,9 +40,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
+import java.io.IOException;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
@@ -55,14 +55,13 @@ public class TestReconstructStripedBlocksWithRackAwareness {
static {
GenericTestUtils.setLogLevel(BlockPlacementPolicy.LOG, Level.ALL);
GenericTestUtils.setLogLevel(BlockManager.blockLog, Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.LOG, Level.ALL);
}
private static final String[] hosts = new String[]{"host1", "host2", "host3",
"host4", "host5", "host6", "host7", "host8", "host9", "host10"};
private static final String[] racks = new String[]{"/r1", "/r1", "/r2", "/r2",
"/r3", "/r3", "/r4", "/r4", "/r5", "/r6"};
- private static final List<String> singleNodeRacks = Arrays.asList("host9", "host10");
- private static final short blockNum = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
@@ -94,6 +93,20 @@ public class TestReconstructStripedBlocksWithRackAwareness {
}
}
+ private MiniDFSCluster.DataNodeProperties stopDataNode(String hostname)
+ throws IOException {
+ MiniDFSCluster.DataNodeProperties dnProp = null;
+ for (int i = 0; i < cluster.getDataNodes().size(); i++) {
+ DataNode dn = cluster.getDataNodes().get(i);
+ if (dn.getDatanodeId().getHostName().equals(hostname)) {
+ dnProp = cluster.stopDataNode(i);
+ cluster.setDataNodeDead(dn.getDatanodeId());
+ LOG.info("stop datanode " + dn.getDatanodeId().getHostName());
+ }
+ }
+ return dnProp;
+ }
+
/**
* When there are all the internal blocks available but they are not placed on
* enough racks, NameNode should avoid normal decoding reconstruction but copy
@@ -102,24 +115,13 @@ public class TestReconstructStripedBlocksWithRackAwareness {
* In this test, we first need to create a scenario that a striped block has
* all the internal blocks but distributed in <6 racks. Then we check if the
* replication monitor can correctly schedule the reconstruction work for it.
- *
- * For the 9 internal blocks + 5 racks setup, the test does the following:
- * 1. create a 6 rack cluster with 10 datanodes, where there are 2 racks only
- * containing 1 datanodes each
- * 2. for a striped block with 9 internal blocks, there must be one internal
- * block locating in a single-node rack. find this node and stop it
- * 3. namenode will trigger reconstruction for the block and since the cluster
- * has only 5 racks remaining, after the reconstruction we have 9 internal
- * blocks distributed in 5 racks.
- * 4. we bring the datanode back, now the cluster has 6 racks again
- * 5. let the datanode call reportBadBlock, this will make the namenode to
- * check if the striped block is placed in >= 6 racks, and the namenode will
- * put the block into the under-replicated queue
- * 6. now we can check if the replication monitor works as expected
*/
@Test
public void testReconstructForNotEnoughRacks() throws Exception {
+ MiniDFSCluster.DataNodeProperties host10 = stopDataNode("host10");
+
final Path file = new Path("/foo");
+ // the file's block is in 9 dn but 5 racks
DFSTestUtil.createFile(fs, file,
BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
@@ -128,39 +130,6 @@ public class TestReconstructStripedBlocksWithRackAwareness {
.getINode4Write(file.toString()).asFile();
BlockInfoStriped blockInfo = (BlockInfoStriped) fileNode.getLastBlock();
- // find the internal block located in the single node rack
- Block internalBlock = null;
- String hostToStop = null;
- for (DatanodeStorageInfo storage : blockInfo.storages) {
- if (singleNodeRacks.contains(storage.getDatanodeDescriptor().getHostName())) {
- hostToStop = storage.getDatanodeDescriptor().getHostName();
- internalBlock = blockInfo.getBlockOnStorage(storage);
- }
- }
- Assert.assertNotNull(internalBlock);
- Assert.assertNotNull(hostToStop);
-
- // delete the block on the chosen datanode
- cluster.corruptBlockOnDataNodesByDeletingBlockFile(
- new ExtendedBlock(bm.getBlockPoolId(), internalBlock));
-
- // stop the chosen datanode
- MiniDFSCluster.DataNodeProperties dnProp = null;
- for (int i = 0; i < cluster.getDataNodes().size(); i++) {
- DataNode dn = cluster.getDataNodes().get(i);
- if (dn.getDatanodeId().getHostName().equals(hostToStop)) {
- dnProp = cluster.stopDataNode(i);
- cluster.setDataNodeDead(dn.getDatanodeId());
- LOG.info("stop datanode " + dn.getDatanodeId().getHostName());
- }
- }
- NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
- Assert.assertEquals(5, topology.getNumOfRacks());
-
- // make sure the reconstruction work can finish
- // now we have 9 internal blocks in 5 racks
- DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
-
// we now should have 9 internal blocks distributed in 5 racks
Set<String> rackSet = new HashSet<>();
for (DatanodeStorageInfo storage : blockInfo.storages) {
@@ -169,27 +138,25 @@ public class TestReconstructStripedBlocksWithRackAwareness {
Assert.assertEquals(5, rackSet.size());
// restart the stopped datanode
- cluster.restartDataNode(dnProp);
+ cluster.restartDataNode(host10);
cluster.waitActive();
// make sure we have 6 racks again
- topology = bm.getDatanodeManager().getNetworkTopology();
+ NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
Assert.assertEquals(hosts.length, topology.getNumOfLeaves());
Assert.assertEquals(6, topology.getNumOfRacks());
// pause all the heartbeats
- DataNode badDn = null;
for (DataNode dn : cluster.getDataNodes()) {
DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
- if (dn.getDatanodeId().getHostName().equals(hostToStop)) {
- badDn = dn;
- }
}
- assert badDn != null;
- // let the DN report the bad block, so that the namenode will put the block
- // into under-replicated queue. note that the block still has 9 internal
- // blocks but in 5 racks
- badDn.reportBadBlocks(new ExtendedBlock(bm.getBlockPoolId(), internalBlock));
+
+ fsn.writeLock();
+ try {
+ bm.processMisReplicatedBlocks();
+ } finally {
+ fsn.writeUnlock();
+ }
// check if replication monitor correctly schedule the replication work
boolean scheduled = false;
@@ -210,4 +177,42 @@ public class TestReconstructStripedBlocksWithRackAwareness {
}
Assert.assertTrue(scheduled);
}
+
+ @Test
+ public void testChooseExcessReplicasToDelete() throws Exception {
+ MiniDFSCluster.DataNodeProperties host10 = stopDataNode("host10");
+
+ final Path file = new Path("/foo");
+ DFSTestUtil.createFile(fs, file,
+ BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+
+ // stop host1
+ MiniDFSCluster.DataNodeProperties host1 = stopDataNode("host1");
+ // bring host10 back
+ cluster.restartDataNode(host10);
+ cluster.waitActive();
+
+ // wait for reconstruction to finish
+ final short blockNum = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+ DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
+
+ // restart host1
+ cluster.restartDataNode(host1);
+ cluster.waitActive();
+ for (DataNode dn : cluster.getDataNodes()) {
+ if (dn.getDatanodeId().getHostName().equals("host1")) {
+ DataNodeTestUtils.triggerBlockReport(dn);
+ break;
+ }
+ }
+
+ // make sure the excess replica is detected, and we delete host1's replica
+ // so that we have 6 racks
+ DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
+ LocatedBlocks blks = fs.getClient().getLocatedBlocks(file.toString(), 0);
+ LocatedStripedBlock block = (LocatedStripedBlock) blks.getLastLocatedBlock();
+ for (DatanodeInfo dn : block.getLocations()) {
+ Assert.assertFalse(dn.getHostName().equals("host1"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 3259612..be63d87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -965,7 +965,8 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
List<DatanodeStorageInfo> first = new ArrayList<>();
List<DatanodeStorageInfo> second = new ArrayList<>();
- replicator.splitNodesWithRack(replicaList, rackMap, first, second);
+ replicator.splitNodesWithRack(replicaList, replicaList, rackMap, first,
+ second);
// storages[0] and storages[1] are in first set as their rack has two
// replica nodes, while storages[2] and dataNodes[5] are in second set.
assertEquals(2, first.size());
@@ -1018,7 +1019,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
DatanodeDescriptor delHintNode = storages[0].getDatanodeDescriptor();
List<StorageType> excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), delHintNode);
assertTrue(excessReplicas.size() == 1);
assertTrue(excessReplicas.contains(storages[0]));
@@ -1031,7 +1032,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
nonExcess.add(excessStorage);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.contains(excessStorage));
@@ -1051,7 +1052,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
nonExcess.add(storages[5]);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(),
storages[5].getDatanodeDescriptor());
assertEquals(1, excessReplicas.size());
@@ -1070,7 +1071,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
nonExcess.add(storages[3]);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[1].getDatanodeDescriptor(),
storages[3].getDatanodeDescriptor());
assertEquals(1, excessReplicas.size());
@@ -1084,7 +1085,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
nonExcess.add(storages[2]);
excessTypes = storagePolicy.chooseExcess((short) 1,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 1,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 1,
excessTypes, storages[2].getDatanodeDescriptor(), null);
assertEquals(1, excessReplicas.size());
assertTrue(excessReplicas.contains(excessSSD));
@@ -1104,7 +1105,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
nonExcess.add(storages[5]);
excessTypes = storagePolicy.chooseExcess((short) 2,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 2,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 2,
excessTypes, null, null);
assertEquals(0, excessReplicas.size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index edcab10..ce210e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -637,7 +637,7 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
List<DatanodeStorageInfo> first = new ArrayList<>();
List<DatanodeStorageInfo> second = new ArrayList<>();
- replicator.splitNodesWithRack(
+ replicator.splitNodesWithRack(replicaList,
replicaList, rackMap, first, second);
assertEquals(3, first.size());
assertEquals(1, second.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f2c80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithUpgradeDomain.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithUpgradeDomain.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithUpgradeDomain.java
index c939220..69bc228 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithUpgradeDomain.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithUpgradeDomain.java
@@ -330,7 +330,7 @@ public class TestReplicationPolicyWithUpgradeDomain
DatanodeDescriptor delHintNode = storages[0].getDatanodeDescriptor();
List<StorageType> excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), delHintNode);
assertTrue(excessReplicas.size() == 1);
assertTrue(excessReplicas.contains(storages[0]));
@@ -340,7 +340,7 @@ public class TestReplicationPolicyWithUpgradeDomain
delHintNode = storages[1].getDatanodeDescriptor();
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), delHintNode);
assertTrue(excessReplicas.size() == 1);
assertTrue(excessReplicas.contains(storages[0]));
@@ -353,7 +353,7 @@ public class TestReplicationPolicyWithUpgradeDomain
nonExcess.add(storages[8]);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[8].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.size() == 1);
assertTrue(excessReplicas.contains(storages[1]));
@@ -366,7 +366,7 @@ public class TestReplicationPolicyWithUpgradeDomain
nonExcess.add(storages[5]);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[8].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.size() == 1);
assertTrue(excessReplicas.contains(storages[1]) ||
@@ -384,7 +384,7 @@ public class TestReplicationPolicyWithUpgradeDomain
nonExcess.add(excessStorage);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.size() == 2);
assertTrue(excessReplicas.contains(storages[0]));
@@ -416,7 +416,7 @@ public class TestReplicationPolicyWithUpgradeDomain
nonExcess.add(storages[8]);
excessTypes = storagePolicy.chooseExcess((short) 3,
DatanodeStorageInfo.toStorageTypes(nonExcess));
- excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(),
storages[7].getDatanodeDescriptor());
assertEquals(1, excessReplicas.size());
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e85174b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e85174b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e85174b
Branch: refs/heads/HDFS-1312
Commit: 8e85174be583f531258bf682bda7df3cbbae728f
Parents: 889dc8b ff0ee84
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 3 17:01:03 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 3 17:01:03 2016 -0800
----------------------------------------------------------------------
dev-support/bin/dist-layout-stitching | 140 ++
dev-support/bin/dist-tar-stitching | 44 +
.../main/resources/checkstyle/checkstyle.xml | 1 +
hadoop-common-project/hadoop-common/CHANGES.txt | 63 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +-
.../apache/hadoop/fs/ChecksumFileSystem.java | 4 +
.../hadoop/fs/CommonConfigurationKeys.java | 14 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 10 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +
.../main/java/org/apache/hadoop/io/IOUtils.java | 4 +-
.../main/java/org/apache/hadoop/io/MD5Hash.java | 11 +
.../hadoop/io/retry/RetryInvocationHandler.java | 22 +-
.../apache/hadoop/io/retry/RetryPolicies.java | 60 +-
.../metrics2/sink/RollingFileSystemSink.java | 143 +-
.../apache/hadoop/security/ProviderUtils.java | 107 +-
.../apache/hadoop/security/SecurityUtil.java | 42 +-
.../hadoop/security/UserGroupInformation.java | 2 +-
.../org/apache/hadoop/util/DataChecksum.java | 12 +-
.../org/apache/hadoop/util/SysInfoWindows.java | 30 +-
.../src/main/resources/core-default.xml | 140 +-
.../src/main/winutils/systeminfo.c | 169 +-
.../src/site/markdown/CommandsManual.md | 4 +-
.../src/site/markdown/GroupsMapping.md | 191 ++
.../src/site/markdown/SecureMode.md | 2 +-
.../src/site/markdown/ServiceLevelAuth.md | 2 +-
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 6 +-
.../hadoop/fs/TestChecksumFileSystem.java | 23 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../apache/hadoop/io/retry/TestRetryProxy.java | 101 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 4 +-
.../hadoop/ipc/TestMultipleProtocolServer.java | 14 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 137 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 1013 +++++------
.../hadoop/ipc/TestRPCServerShutdown.java | 106 ++
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 295 ++++
.../sink/RollingFileSystemSinkTestBase.java | 7 +-
.../sink/TestRollingFileSystemSink.java | 10 +-
.../hadoop/service/TestCompositeService.java | 480 ++++-
.../apache/hadoop/test/GenericTestUtils.java | 56 +
.../apache/hadoop/util/TestSysInfoWindows.java | 19 +-
.../hadoop-common/src/test/proto/test.proto | 33 +
.../src/test/proto/test_rpc_service.proto | 7 +
hadoop-dist/pom.xml | 148 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 255 +--
.../org/apache/hadoop/hdfs/DFSInputStream.java | 72 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 49 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 85 +
.../hadoop/hdfs/DistributedFileSystem.java | 27 +-
.../apache/hadoop/hdfs/FileChecksumHelper.java | 416 +++++
.../hdfs/client/HdfsClientConfigKeys.java | 7 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 13 +
.../protocol/datatransfer/IOStreamPair.java | 11 +-
.../ClientNamenodeProtocolTranslatorPB.java | 12 +
.../hadoop/hdfs/web/URLConnectionFactory.java | 30 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 9 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 65 +
.../bkjournal/TestBookKeeperAsHASharedDir.java | 46 +-
.../src/test/resources/log4j.properties | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 3 -
...tNamenodeProtocolServerSideTranslatorPB.java | 17 +
.../hadoop/hdfs/server/balancer/Balancer.java | 22 +-
.../server/blockmanagement/BlockManager.java | 139 +-
.../blockmanagement/BlockPlacementPolicy.java | 48 +-
.../BlockPlacementPolicyDefault.java | 20 +-
.../blockmanagement/ErasureCodingWork.java | 6 +
.../blockmanagement/ExcessReplicaMap.java | 106 ++
.../hdfs/server/datanode/BPOfferService.java | 17 +-
.../hdfs/server/datanode/BPServiceActor.java | 19 +-
.../server/datanode/BlockChecksumHelper.java | 254 +++
.../hdfs/server/datanode/BlockReceiver.java | 8 +-
.../hdfs/server/datanode/BlockSender.java | 1 -
.../hadoop/hdfs/server/datanode/DNConf.java | 4 +
.../hadoop/hdfs/server/datanode/DataNode.java | 52 +-
.../server/datanode/DataNodeLayoutVersion.java | 5 +-
.../hdfs/server/datanode/DataStorage.java | 11 +-
.../hdfs/server/datanode/DataXceiver.java | 176 +-
.../hdfs/server/datanode/DatanodeUtil.java | 4 +-
.../datanode/IncrementalBlockReportManager.java | 51 +-
.../erasurecode/ErasureCodingWorker.java | 50 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +
.../hdfs/server/namenode/FSDirAttrOp.java | 74 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 109 +-
.../hdfs/server/namenode/FSEditLogAsync.java | 322 ++++
.../hdfs/server/namenode/FSEditLogOp.java | 213 +--
.../hdfs/server/namenode/FSEditLogOpCodes.java | 108 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 2 +-
.../server/namenode/FSImageFormatPBINode.java | 12 +-
.../hdfs/server/namenode/FSNamesystem.java | 22 +
.../hadoop/hdfs/server/namenode/NameNode.java | 1 -
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +
.../hdfs/server/namenode/NamenodeFsck.java | 6 +-
.../namenode/metrics/NameNodeMetrics.java | 4 +-
.../protocol/StorageReceivedDeletedBlocks.java | 7 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 48 +-
.../OfflineImageReconstructor.java | 1639 ++++++++++++++++++
.../OfflineImageViewerPB.java | 22 +-
.../offlineImageViewer/PBImageXmlWriter.java | 248 ++-
.../src/main/resources/hdfs-default.xml | 135 +-
.../src/site/markdown/ArchivalStorage.md | 15 +
.../src/site/markdown/HdfsPermissionsGuide.md | 8 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 28 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 3 +
.../hadoop/hdfs/TestApplyingStoragePolicy.java | 268 +++
.../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 29 +-
.../hadoop/hdfs/TestDatanodeRegistration.java | 6 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 8 +
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
.../hdfs/server/balancer/TestBalancer.java | 125 +-
.../server/blockmanagement/TestNodeCount.java | 7 +-
.../TestOverReplicatedBlocks.java | 10 +-
...constructStripedBlocksWithRackAwareness.java | 133 +-
.../blockmanagement/TestReplicationPolicy.java | 15 +-
.../TestReplicationPolicyWithNodeGroup.java | 2 +-
.../TestReplicationPolicyWithUpgradeDomain.java | 12 +-
.../server/datanode/SimulatedFSDataset.java | 11 +-
.../server/datanode/TestBPOfferService.java | 2 +-
.../hdfs/server/datanode/TestBatchIbr.java | 263 +++
.../server/datanode/TestDataNodeMetrics.java | 48 +-
.../TestDataXceiverLazyPersistHint.java | 6 +-
.../datanode/TestIncrementalBlockReports.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 4 +-
.../impl/TestInterDatanodeProtocol.java | 6 +-
.../hdfs/server/namenode/TestAuditLogs.java | 15 +-
.../hdfs/server/namenode/TestEditLog.java | 59 +-
.../server/namenode/TestEditLogAutoroll.java | 26 +
.../namenode/TestEditLogJournalFailures.java | 35 +-
.../hdfs/server/namenode/TestEditLogRace.java | 144 +-
.../server/namenode/TestFSEditLogLoader.java | 37 +-
.../server/namenode/TestNameNodeRecovery.java | 31 +-
.../server/namenode/ha/TestEditLogTailer.java | 39 +-
.../namenode/metrics/TestNameNodeMetrics.java | 2 +-
.../hdfs/tools/TestStoragePolicyCommands.java | 63 +
.../TestOfflineImageViewer.java | 56 +-
.../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 37 +-
.../sink/TestRollingFileSystemSinkWithHdfs.java | 34 +-
...TestRollingFileSystemSinkWithSecureHdfs.java | 3 +-
.../apache/hadoop/security/TestPermission.java | 13 +
.../resources/hadoop-56-layout-datanode-dir.tgz | Bin 0 -> 198996 bytes
.../resources/hadoop-to-57-dn-layout-dir.txt | 24 +
hadoop-mapreduce-project/CHANGES.txt | 5 +
.../hadoop/mapreduce/v2/app/TestMRApp.java | 104 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 8 +-
.../src/main/resources/mapred-default.xml | 27 +-
.../mapreduce/v2/hs/CachedHistoryStorage.java | 146 +-
.../v2/hs/webapp/dao/AMAttemptInfo.java | 4 +-
.../hadoop/mapreduce/v2/hs/TestJobHistory.java | 255 ++-
.../v2/hs/webapp/TestHsWebServicesJobs.java | 11 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 47 +-
.../hadoop/fs/s3a/TestS3AConfiguration.java | 34 +
.../hadoop/fs/azure/SimpleKeyProvider.java | 5 +-
.../hadoop-azure/src/site/markdown/index.md | 6 +-
.../fs/azure/TestWasbUriAndConfiguration.java | 35 +
.../src/site/markdown/DistCp.md.vm | 13 +-
hadoop-tools/hadoop-openstack/pom.xml | 4 +-
hadoop-yarn-project/CHANGES.txt | 33 +
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +
.../yarn/webapp/log/AggregatedLogsBlock.java | 27 +
.../src/main/resources/yarn-default.xml | 10 +
.../logaggregation/TestAggregatedLogsBlock.java | 20 +
...pplicationHistoryManagerOnTimelineStore.java | 9 +
.../nodemanager/DefaultContainerExecutor.java | 32 +-
.../logaggregation/AppLogAggregatorImpl.java | 68 +-
.../impl/container-executor.c | 54 +-
.../test/test-container-executor.c | 99 +-
.../TestDefaultContainerExecutor.java | 21 +-
.../server/nodemanager/TestNMAuditLogger.java | 40 +-
.../TestLogAggregationService.java | 59 +
.../resourcemanager/NodesListManager.java | 13 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 4 +
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../scheduler/SchedulerNode.java | 172 +-
.../scheduler/SchedulerNodeReport.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 18 +-
.../scheduler/capacity/ParentQueue.java | 2 +-
.../allocator/IncreaseContainerAllocator.java | 4 +-
.../allocator/RegularContainerAllocator.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 6 +-
.../scheduler/fair/FairScheduler.java | 8 +-
.../scheduler/fifo/FifoScheduler.java | 20 +-
.../resourcemanager/TestRMAuditLogger.java | 44 +-
.../resourcemanager/TestRMNodeTransitions.java | 14 +
.../resourcemanager/TestResourceManager.java | 7 +
.../TestWorkPreservingRMRestart.java | 12 +-
.../scheduler/TestAbstractYarnScheduler.java | 4 +-
.../capacity/TestCapacityScheduler.java | 39 +-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 4 +-
.../capacity/TestContainerResizing.java | 2 +-
.../capacity/TestIncreaseAllocationExpirer.java | 2 +-
.../scheduler/capacity/TestLeafQueue.java | 16 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestReservations.java | 196 +--
.../scheduler/capacity/TestUtils.java | 2 +-
.../scheduler/fair/TestFSAppAttempt.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 6 +-
.../TestMiniYarnClusterNodeUtilization.java | 6 +-
204 files changed, 9587 insertions(+), 2682 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e85174b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e85174b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
[14/50] [abbrv] hadoop git commit: HDFS-9864. Correct reference for
RENEWDELEGATIONTOKEN and CANCELDELEGATIONTOKEN in webhdfs doc. Contributed by
Brahma Reddy Battula.
Posted by ar...@apache.org.
HDFS-9864. Correct reference for RENEWDELEGATIONTOKEN and CANCELDELEGATIONTOKEN in webhdfs doc. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bc023b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bc023b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bc023b3
Branch: refs/heads/HDFS-1312
Commit: 8bc023b3b17754bf422eb9a8e749e8ea01768ac2
Parents: 809ebc0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 29 18:40:28 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 29 18:42:21 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc023b3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 78d2721..3d57efa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2949,6 +2949,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9765. TestBlockScanner#testVolumeIteratorWithCaching fails
intermittently. (aajisaka)
+ HDFS-9864. Correct reference for RENEWDELEGATIONTOKEN and
+ CANCELDELEGATIONTOKEN in webhdfs doc. (Brahma Reddy Battula via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc023b3/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 2d3d361..79e79b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -163,8 +163,8 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`SETOWNER`](#Set_Owner) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setOwner)
* [`SETPERMISSION`](#Set_Permission) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setPermission)
* [`SETTIMES`](#Set_Access_or_Modification_Time) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setTimes)
- * [`RENEWDELEGATIONTOKEN`](#Renew_Delegation_Token) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renewDelegationToken)
- * [`CANCELDELEGATIONTOKEN`](#Cancel_Delegation_Token) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).cancelDelegationToken)
+ * [`RENEWDELEGATIONTOKEN`](#Renew_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).renewDelegationToken)
+ * [`CANCELDELEGATIONTOKEN`](#Cancel_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).cancelDelegationToken)
* [`CREATESNAPSHOT`](#Create_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).createSnapshot)
* [`RENAMESNAPSHOT`](#Rename_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renameSnapshot)
* [`SETXATTR`](#Set_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setXAttr)
@@ -1071,7 +1071,7 @@ See also: [`renewer`](#Renewer), [FileSystem](../../api/org/apache/hadoop/fs/Fil
{"long": 1320962673997} //the new expiration time
-See also: [`token`](#Token), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renewDelegationToken
+See also: [`token`](#Token), [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).renewDelegationToken
### Cancel Delegation Token
@@ -1084,7 +1084,7 @@ See also: [`token`](#Token), [FileSystem](../../api/org/apache/hadoop/fs/FileSys
HTTP/1.1 200 OK
Content-Length: 0
-See also: [`token`](#Token), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).cancelDelegationToken
+See also: [`token`](#Token), [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).cancelDelegationToken
Error Responses
---------------
[21/50] [abbrv] hadoop git commit: YARN-4704.
TestResourceManager#testResourceAllocation() fails when using FairScheduler.
(Yufei Gu via kasha)
Posted by ar...@apache.org.
YARN-4704. TestResourceManager#testResourceAllocation() fails when using FairScheduler. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9dafaaaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9dafaaaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9dafaaaf
Branch: refs/heads/HDFS-1312
Commit: 9dafaaaf0de68ce7f5e495ea4b8e0ce036dc35a2
Parents: 2151716
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Feb 29 16:09:53 2016 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Feb 29 16:10:12 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/TestResourceManager.java | 7 +++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9dafaaaf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 27eff2d..88396af 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -242,6 +242,9 @@ Release 2.9.0 - UNRELEASED
YARN-4731. container-executor should not follow symlinks in
recursive_unlink_children (Colin Patrick McCabe via jlowe)
+ YARN-4704. TestResourceManager#testResourceAllocation() fails when using
+ FairScheduler. (Yufei Gu via kasha)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9dafaaaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index 9ceeffb..3b59417 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -57,6 +58,8 @@ public class TestResourceManager {
@Before
public void setUp() throws Exception {
Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getCanonicalName());
UserGroupInformation.setConfiguration(conf);
resourceManager = new ResourceManager();
resourceManager.init(conf);
@@ -261,6 +264,8 @@ public class TestResourceManager {
}
};
Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getCanonicalName());
conf.set(filterInitializerConfKey, filterInitializer);
conf.set("hadoop.security.authentication", "kerberos");
conf.set("hadoop.http.authentication.type", "kerberos");
@@ -295,6 +300,8 @@ public class TestResourceManager {
for (String filterInitializer : simpleFilterInitializers) {
resourceManager = new ResourceManager();
Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getCanonicalName());
conf.set(filterInitializerConfKey, filterInitializer);
try {
UserGroupInformation.setConfiguration(conf);
[13/50] [abbrv] hadoop git commit: HADOOP-12552. Fix
undeclared/unused dependency to httpclient (iwasakims)
Posted by ar...@apache.org.
HADOOP-12552. Fix undeclared/unused dependency to httpclient (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/809ebc0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/809ebc0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/809ebc0b
Branch: refs/heads/HDFS-1312
Commit: 809ebc0b146135d86433e9c7bfa17e294b7928f2
Parents: e3ac231
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Mon Feb 29 18:20:14 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Mon Feb 29 18:20:14 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/809ebc0b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 64ec531..eb33464 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -733,6 +733,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12416. Trash messages should be handled by Logger instead of being
delivered on System.out. (Mingliang Liu via aajisaka)
+ HADOOP-12552. Fix undeclared/unused dependency to httpclient (iwasakims)
+
NEW FEATURES
HADOOP-11226. Add a configuration to set ipc.Client's traffic class with
http://git-wip-us.apache.org/repos/asf/hadoop/blob/809ebc0b/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 7e4d090..503f312 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -66,8 +66,8 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/809ebc0b/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index 1b541e2..542a523 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -113,8 +113,8 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
<scope>compile</scope>
</dependency>
[40/50] [abbrv] hadoop git commit: HDFS-9835. OIV: add ReverseXML
processor which reconstructs an fsimage from an XML file (cmccabe)
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index d23ff61..71dc939 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -23,17 +23,24 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.RandomAccessFile;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
+import java.util.TimeZone;
+import com.google.protobuf.ByteString;
+import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
@@ -55,6 +62,14 @@ import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.util.LimitInputStream;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.apache.hadoop.util.VersionInfo;
+
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_MASK;
/**
* PBImageXmlWriter walks over an fsimage structure and writes out
@@ -64,11 +79,20 @@ import com.google.common.collect.Lists;
public final class PBImageXmlWriter {
private final Configuration conf;
private final PrintStream out;
+ private final SimpleDateFormat isoDateFormat;
private String[] stringTable;
+ public static SimpleDateFormat createSimpleDateFormat() {
+ SimpleDateFormat format =
+ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return format;
+ }
+
public PBImageXmlWriter(Configuration conf, PrintStream out) {
this.conf = conf;
this.out = out;
+ this.isoDateFormat = createSimpleDateFormat();
}
public void visit(RandomAccessFile file) throws IOException {
@@ -80,6 +104,16 @@ public final class PBImageXmlWriter {
try (FileInputStream fin = new FileInputStream(file.getFD())) {
out.print("<?xml version=\"1.0\"?>\n<fsimage>");
+ out.print("<version>");
+ o("layoutVersion", summary.getLayoutVersion());
+ o("onDiskVersion", summary.getOndiskVersion());
+ // Output the version of OIV (which is not necessarily the version of
+ // the fsimage file). This could be helpful in the case where a bug
+ // in OIV leads to information loss in the XML-- we can quickly tell
+ // if a specific fsimage XML file is affected by this bug.
+ o("oivRevision", VersionInfo.getRevision());
+ out.print("</version>\n");
+
ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary
.getSectionsList());
Collections.sort(sections, new Comparator<FileSummary.Section>() {
@@ -146,6 +180,8 @@ public final class PBImageXmlWriter {
out.print("<CacheManagerSection>");
CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(is);
o("nextDirectiveId", s.getNextDirectiveId());
+ o("numDirectives", s.getNumDirectives());
+ o("numPools", s.getNumPools());
for (int i = 0; i < s.getNumPools(); ++i) {
CachePoolInfoProto p = CachePoolInfoProto.parseDelimitedFrom(is);
out.print("<pool>");
@@ -163,7 +199,7 @@ public final class PBImageXmlWriter {
.o("replication", p.getReplication()).o("pool", p.getPool());
out.print("<expiration>");
CacheDirectiveInfoExpirationProto e = p.getExpiration();
- o("millis", e.getMillis()).o("relatilve", e.getIsRelative());
+ o("millis", e.getMillis()).o("relative", e.getIsRelative());
out.print("</expiration>\n");
out.print("</directive>\n");
}
@@ -187,13 +223,48 @@ public final class PBImageXmlWriter {
out.print("</FileUnderConstructionSection>\n");
}
+ private void dumpXattrs(INodeSection.XAttrFeatureProto xattrs) {
+ out.print("<xattrs>");
+ for (INodeSection.XAttrCompactProto xattr : xattrs.getXAttrsList()) {
+ out.print("<xattr>");
+ int encodedName = xattr.getName();
+ int ns = (XATTR_NAMESPACE_MASK & (encodedName >> XATTR_NAMESPACE_OFFSET)) |
+ ((XATTR_NAMESPACE_EXT_MASK & (encodedName >> XATTR_NAMESPACE_EXT_OFFSET)) << 2);
+ o("ns", XAttrProtos.XAttrProto.
+ XAttrNamespaceProto.valueOf(ns).toString());
+ o("name", stringTable[XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET)]);
+ ByteString val = xattr.getValue();
+ if (val.isValidUtf8()) {
+ o("val", val.toStringUtf8());
+ } else {
+ o("valHex", Hex.encodeHexString(val.toByteArray()));
+ }
+ out.print("</xattr>");
+ }
+ out.print("</xattrs>");
+ }
+
private void dumpINodeDirectory(INodeDirectory d) {
o("mtime", d.getModificationTime()).o("permission",
dumpPermission(d.getPermission()));
+ if (d.hasXAttrs()) {
+ dumpXattrs(d.getXAttrs());
+ }
dumpAcls(d.getAcl());
if (d.hasDsQuota() && d.hasNsQuota()) {
o("nsquota", d.getNsQuota()).o("dsquota", d.getDsQuota());
}
+ INodeSection.QuotaByStorageTypeFeatureProto typeQuotas =
+ d.getTypeQuotas();
+ if (typeQuotas != null) {
+ for (INodeSection.QuotaByStorageTypeEntryProto entry:
+ typeQuotas.getQuotasList()) {
+ out.print("<typeQuota>");
+ o("type", entry.getStorageType().toString());
+ o("quota", entry.getQuota());
+ out.print("</typeQuota>");
+ }
+ }
}
private void dumpINodeDirectorySection(InputStream in) throws IOException {
@@ -208,10 +279,10 @@ public final class PBImageXmlWriter {
out.print("<directory>");
o("parent", e.getParent());
for (long id : e.getChildrenList()) {
- o("inode", id);
+ o("child", id);
}
for (int refId : e.getRefChildrenList()) {
- o("inodereference-index", refId);
+ o("refChild", refId);
}
out.print("</directory>\n");
}
@@ -244,6 +315,9 @@ public final class PBImageXmlWriter {
.o("atime", f.getAccessTime())
.o("preferredBlockSize", f.getPreferredBlockSize())
.o("permission", dumpPermission(f.getPermission()));
+ if (f.hasXAttrs()) {
+ dumpXattrs(f.getXAttrs());
+ }
dumpAcls(f.getAcl());
if (f.getBlocksCount() > 0) {
out.print("<blocks>");
@@ -255,6 +329,12 @@ public final class PBImageXmlWriter {
}
out.print("</blocks>\n");
}
+ if (f.hasStoragePolicyID()) {
+ o("storagePolicyId", f.getStoragePolicyID());
+ }
+ if (f.getIsStriped()) {
+ out.print("<isStriped/>");
+ }
if (f.hasFileUC()) {
INodeSection.FileUnderConstructionFeature u = f.getFileUC();
@@ -281,25 +361,28 @@ public final class PBImageXmlWriter {
INodeSection s = INodeSection.parseDelimitedFrom(in);
out.print("<INodeSection>");
o("lastInodeId", s.getLastInodeId());
+ o("numInodes", s.getNumInodes());
for (int i = 0; i < s.getNumInodes(); ++i) {
INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
out.print("<inode>");
- o("id", p.getId()).o("type", p.getType()).o("name",
- p.getName().toStringUtf8());
-
- if (p.hasFile()) {
- dumpINodeFile(p.getFile());
- } else if (p.hasDirectory()) {
- dumpINodeDirectory(p.getDirectory());
- } else if (p.hasSymlink()) {
- dumpINodeSymlink(p.getSymlink());
- }
-
+ dumpINodeFields(p);
out.print("</inode>\n");
}
out.print("</INodeSection>\n");
}
+ private void dumpINodeFields(INodeSection.INode p) {
+ o("id", p.getId()).o("type", p.getType()).o("name",
+ p.getName().toStringUtf8());
+ if (p.hasFile()) {
+ dumpINodeFile(p.getFile());
+ } else if (p.hasDirectory()) {
+ dumpINodeDirectory(p.getDirectory());
+ } else if (p.hasSymlink()) {
+ dumpINodeSymlink(p.getSymlink());
+ }
+ }
+
private void dumpINodeSymlink(INodeSymlink s) {
o("permission", dumpPermission(s.getPermission()))
.o("target", s.getTarget().toStringUtf8())
@@ -308,7 +391,8 @@ public final class PBImageXmlWriter {
private void dumpNameSection(InputStream in) throws IOException {
NameSystemSection s = NameSystemSection.parseDelimitedFrom(in);
- out.print("<NameSection>\n");
+ out.print("<NameSection>");
+ o("namespaceId", s.getNamespaceId());
o("genstampV1", s.getGenstampV1()).o("genstampV2", s.getGenstampV2())
.o("genstampV1Limit", s.getGenstampV1Limit())
.o("lastAllocatedBlockId", s.getLastAllocatedBlockId())
@@ -317,18 +401,73 @@ public final class PBImageXmlWriter {
}
private String dumpPermission(long permission) {
- return FSImageFormatPBINode.Loader.loadPermission(permission, stringTable)
- .toString();
+ PermissionStatus permStatus = FSImageFormatPBINode.Loader.
+ loadPermission(permission, stringTable);
+ return String.format("%s:%s:%04o", permStatus.getUserName(),
+ permStatus.getGroupName(), permStatus.getPermission().toExtendedShort());
}
private void dumpSecretManagerSection(InputStream is) throws IOException {
out.print("<SecretManagerSection>");
SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(is);
+ int expectedNumDelegationKeys = s.getNumKeys();
+ int expectedNumTokens = s.getNumTokens();
o("currentId", s.getCurrentId()).o("tokenSequenceNumber",
- s.getTokenSequenceNumber());
+ s.getTokenSequenceNumber()).
+ o("numDelegationKeys", expectedNumDelegationKeys).
+ o("numTokens", expectedNumTokens);
+ for (int i = 0; i < expectedNumDelegationKeys; i++) {
+ SecretManagerSection.DelegationKey dkey =
+ SecretManagerSection.DelegationKey.parseDelimitedFrom(is);
+ out.print("<delegationKey>");
+ o("id", dkey.getId());
+ o("key", Hex.encodeHexString(dkey.getKey().toByteArray()));
+ if (dkey.hasExpiryDate()) {
+ dumpDate("expiry", dkey.getExpiryDate());
+ }
+ out.print("</delegationKey>");
+ }
+ for (int i = 0; i < expectedNumTokens; i++) {
+ SecretManagerSection.PersistToken token =
+ SecretManagerSection.PersistToken.parseDelimitedFrom(is);
+ out.print("<token>");
+ if (token.hasVersion()) {
+ o("version", token.getVersion());
+ }
+ if (token.hasOwner()) {
+ o("owner", token.getOwner());
+ }
+ if (token.hasRenewer()) {
+ o("renewer", token.getRenewer());
+ }
+ if (token.hasRealUser()) {
+ o("realUser", token.getRealUser());
+ }
+ if (token.hasIssueDate()) {
+ dumpDate("issueDate", token.getIssueDate());
+ }
+ if (token.hasMaxDate()) {
+ dumpDate("maxDate", token.getMaxDate());
+ }
+ if (token.hasSequenceNumber()) {
+ o("sequenceNumber", token.getSequenceNumber());
+ }
+ if (token.hasMasterKeyId()) {
+ o("masterKeyId", token.getMasterKeyId());
+ }
+ if (token.hasExpiryDate()) {
+ dumpDate("expiryDate", token.getExpiryDate());
+ }
+ out.print("</token>");
+ }
out.print("</SecretManagerSection>");
}
+ private void dumpDate(String tag, long date) {
+ out.print("<" + tag + ">" +
+ isoDateFormat.format(new Date(date)) + "</" + tag + ">");
+ }
+
private void dumpSnapshotDiffSection(InputStream in) throws IOException {
out.print("<SnapshotDiffSection>");
while (true) {
@@ -337,30 +476,46 @@ public final class PBImageXmlWriter {
if (e == null) {
break;
}
- out.print("<diff>");
- o("inodeid", e.getInodeId());
+ switch (e.getType()) {
+ case FILEDIFF:
+ out.print("<fileDiffEntry>");
+ break;
+ case DIRECTORYDIFF:
+ out.print("<dirDiffEntry>");
+ break;
+ default:
+ throw new IOException("unknown DiffEntry type " + e.getType());
+ }
+ o("inodeId", e.getInodeId());
+ o("count", e.getNumOfDiff());
switch (e.getType()) {
case FILEDIFF: {
for (int i = 0; i < e.getNumOfDiff(); ++i) {
- out.print("<filediff>");
+ out.print("<fileDiff>");
SnapshotDiffSection.FileDiff f = SnapshotDiffSection.FileDiff
.parseDelimitedFrom(in);
o("snapshotId", f.getSnapshotId()).o("size", f.getFileSize()).o(
"name", f.getName().toStringUtf8());
- out.print("</filediff>\n");
+ out.print("</fileDiff>\n");
}
}
break;
case DIRECTORYDIFF: {
for (int i = 0; i < e.getNumOfDiff(); ++i) {
- out.print("<dirdiff>");
+ out.print("<dirDiff>");
SnapshotDiffSection.DirectoryDiff d = SnapshotDiffSection.DirectoryDiff
.parseDelimitedFrom(in);
o("snapshotId", d.getSnapshotId())
- .o("isSnapshotroot", d.getIsSnapshotRoot())
.o("childrenSize", d.getChildrenSize())
- .o("name", d.getName().toStringUtf8());
-
+ .o("isSnapshotRoot", d.getIsSnapshotRoot())
+ .o("name", d.getName().toStringUtf8())
+ .o("createdListSize", d.getCreatedListSize());
+ for (long did : d.getDeletedINodeList()) {
+ o("deletedInode", did);
+ }
+ for (int dRefid : d.getDeletedINodeRefList()) {
+ o("deletedInoderef", dRefid);
+ }
for (int j = 0; j < d.getCreatedListSize(); ++j) {
SnapshotDiffSection.CreatedListEntry ce = SnapshotDiffSection.CreatedListEntry
.parseDelimitedFrom(in);
@@ -368,24 +523,23 @@ public final class PBImageXmlWriter {
o("name", ce.getName().toStringUtf8());
out.print("</created>\n");
}
- for (long did : d.getDeletedINodeList()) {
- out.print("<deleted>");
- o("inode", did);
- out.print("</deleted>\n");
- }
- for (int dRefid : d.getDeletedINodeRefList()) {
- out.print("<deleted>");
- o("inodereference-index", dRefid);
- out.print("</deleted>\n");
- }
- out.print("</dirdiff>\n");
+ out.print("</dirDiff>\n");
}
- }
break;
+ }
default:
break;
}
- out.print("</diff>");
+ switch (e.getType()) {
+ case FILEDIFF:
+ out.print("</fileDiffEntry>");
+ break;
+ case DIRECTORYDIFF:
+ out.print("</dirDiffEntry>");
+ break;
+ default:
+ throw new IOException("unknown DiffEntry type " + e.getType());
+ }
}
out.print("</SnapshotDiffSection>\n");
}
@@ -394,6 +548,7 @@ public final class PBImageXmlWriter {
out.print("<SnapshotSection>");
SnapshotSection s = SnapshotSection.parseDelimitedFrom(in);
o("snapshotCounter", s.getSnapshotCounter());
+ o("numSnapshots", s.getNumSnapshots());
if (s.getSnapshottableDirCount() > 0) {
out.print("<snapshottableDir>");
for (long id : s.getSnapshottableDirList()) {
@@ -404,7 +559,12 @@ public final class PBImageXmlWriter {
for (int i = 0; i < s.getNumSnapshots(); ++i) {
SnapshotSection.Snapshot pbs = SnapshotSection.Snapshot
.parseDelimitedFrom(in);
- o("snapshot", pbs.getSnapshotId());
+ out.print("<snapshot>");
+ o("id", pbs.getSnapshotId());
+ out.print("<root>");
+ dumpINodeFields(pbs.getRoot());
+ out.print("</root>");
+ out.print("</snapshot>");
}
out.print("</SnapshotSection>\n");
}
@@ -420,6 +580,14 @@ public final class PBImageXmlWriter {
}
private PBImageXmlWriter o(final String e, final Object v) {
+ if (v instanceof Boolean) {
+ // For booleans, the presence of the element indicates true, and its
+ // absence indicates false.
+ if ((Boolean)v != false) {
+ out.print("<" + e + "/>");
+ }
+ return this;
+ }
out.print("<" + e + ">" +
XMLUtils.mangleXmlString(v.toString(), true) + "</" + e + ">");
return this;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 91b79e2..c7a6ae9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -47,6 +47,8 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -66,6 +68,8 @@ import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -89,8 +93,7 @@ public class TestOfflineImageViewer {
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
static int dirCount = 0;
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
+ private static File tempDir;
// Create a populated namespace for later testing. Save its contents to a
// data structure and store its fsimage location.
@@ -98,6 +101,7 @@ public class TestOfflineImageViewer {
// multiple tests.
@BeforeClass
public static void createOriginalFSImage() throws IOException {
+ tempDir = Files.createTempDir();
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
@@ -169,6 +173,9 @@ public class TestOfflineImageViewer {
hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
// OIV should be able to handle empty value XAttrs
hdfs.setXAttr(xattr, "user.a3", null);
+ // OIV should be able to handle XAttr values that can't be expressed
+ // as UTF8
+ hdfs.setXAttr(xattr, "user.a4", new byte[]{ -0x3d, 0x28 });
writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));
// Write results to the fsimage file
@@ -190,6 +197,7 @@ public class TestOfflineImageViewer {
@AfterClass
public static void deleteOriginalFSImage() throws IOException {
+ FileUtils.deleteQuietly(tempDir);
if (originalFsimage != null && originalFsimage.exists()) {
originalFsimage.delete();
}
@@ -204,7 +212,7 @@ public class TestOfflineImageViewer {
@Test(expected = IOException.class)
public void testTruncatedFSImage() throws IOException {
- File truncatedFile = folder.newFile();
+ File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
new FileDistributionCalculator(new Configuration(), 0, 0, output)
@@ -449,4 +457,46 @@ public class TestOfflineImageViewer {
connection.connect();
assertEquals(expectedCode, connection.getResponseCode());
}
+
+ /**
+ * Tests the ReverseXML processor.
+ *
+ * 1. Translate fsimage -> reverseImage.xml
+ * 2. Translate reverseImage.xml -> reverseImage
+ * 3. Translate reverseImage -> reverse2Image.xml
+ * 4. Verify that reverseImage.xml and reverse2Image.xml match
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testReverseXmlRoundTrip() throws Throwable {
+ GenericTestUtils.setLogLevel(OfflineImageReconstructor.LOG,
+ Level.TRACE);
+ File reverseImageXml = new File(tempDir, "reverseImage.xml");
+ File reverseImage = new File(tempDir, "reverseImage");
+ File reverseImage2Xml = new File(tempDir, "reverseImage2.xml");
+ LOG.info("Creating reverseImage.xml=" + reverseImageXml.getAbsolutePath() +
+ ", reverseImage=" + reverseImage.getAbsolutePath() +
+ ", reverseImage2Xml=" + reverseImage2Xml.getAbsolutePath());
+ if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+ "-i", originalFsimage.getAbsolutePath(),
+ "-o", reverseImageXml.getAbsolutePath() }) != 0) {
+ throw new IOException("oiv returned failure creating first XML file.");
+ }
+ if (OfflineImageViewerPB.run(new String[] { "-p", "ReverseXML",
+ "-i", reverseImageXml.getAbsolutePath(),
+ "-o", reverseImage.getAbsolutePath() }) != 0) {
+ throw new IOException("oiv returned failure recreating fsimage file.");
+ }
+ if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+ "-i", reverseImage.getAbsolutePath(),
+ "-o", reverseImage2Xml.getAbsolutePath() }) != 0) {
+ throw new IOException("oiv returned failure creating second " +
+ "XML file.");
+ }
+ // The XML file we wrote based on the re-created fsimage should be the
+ // same as the one we dumped from the original fsimage.
+ Assert.assertEquals("",
+ GenericTestUtils.getFilesDiff(reverseImageXml, reverseImage2Xml));
+ }
}
[15/50] [abbrv] hadoop git commit: YARN-4731. container-executor
should not follow symlinks in recursive_unlink_children. Contributed by Colin
Patrick McCabe
Posted by ar...@apache.org.
YARN-4731. container-executor should not follow symlinks in recursive_unlink_children. Contributed by Colin Patrick McCabe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c58a6d53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c58a6d53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c58a6d53
Branch: refs/heads/HDFS-1312
Commit: c58a6d53c58209a8f78ff64e04e9112933489fb5
Parents: 8bc023b
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Feb 29 15:24:35 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Feb 29 15:24:35 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../impl/container-executor.c | 54 ++++++++++-
.../test/test-container-executor.c | 99 +++++++++++++++++++-
3 files changed, 153 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c58a6d53/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3c91fbd..27eff2d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -239,6 +239,9 @@ Release 2.9.0 - UNRELEASED
YARN-4566. Fix test failure in TestMiniYarnClusterNodeUtilization.
(Takashi Ohnishi via rohithsharmaks)
+ YARN-4731. container-executor should not follow symlinks in
+ recursive_unlink_children (Colin Patrick McCabe via jlowe)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c58a6d53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 4bc8c78..44de2bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1579,9 +1579,9 @@ static int rmdir_as_nm(const char* path) {
static int open_helper(int dirfd, const char *name) {
int fd;
if (dirfd >= 0) {
- fd = openat(dirfd, name, O_RDONLY);
+ fd = openat(dirfd, name, O_RDONLY | O_NOFOLLOW);
} else {
- fd = open(name, O_RDONLY);
+ fd = open(name, O_RDONLY | O_NOFOLLOW);
}
if (fd >= 0) {
return fd;
@@ -1615,6 +1615,34 @@ static int unlink_helper(int dirfd, const char *name, int flags) {
return errno;
}
+/**
+ * Determine if an entry in a directory is a symlink.
+ *
+ * @param dirfd The directory file descriptor, or -1 if there is none.
+ * @param name If dirfd is -1, this is the path to examine.
+ * Otherwise, this is the file name in the directory to
+ * examine.
+ *
+ * @return 0 if the entry is not a symlink
+ * 1 if the entry is a symlink
+ * A negative errno code if we couldn't access the entry.
+ */
+static int is_symlink_helper(int dirfd, const char *name)
+{
+ struct stat stat;
+
+ if (dirfd < 0) {
+ if (lstat(name, &stat) < 0) {
+ return -errno;
+ }
+ } else {
+ if (fstatat(dirfd, name, &stat, AT_SYMLINK_NOFOLLOW) < 0) {
+ return -errno;
+ }
+ }
+ return !!S_ISLNK(stat.st_mode);
+}
+
static int recursive_unlink_helper(int dirfd, const char *name,
const char* fullpath)
{
@@ -1622,6 +1650,28 @@ static int recursive_unlink_helper(int dirfd, const char *name,
DIR *dfd = NULL;
struct stat stat;
+ // Check to see if the file is a symlink. If so, delete the symlink rather
+ // than what it points to.
+ ret = is_symlink_helper(dirfd, name);
+ if (ret < 0) {
+ // is_symlink_helper failed.
+ ret = -ret;
+ fprintf(LOGFILE, "is_symlink_helper(%s) failed: %s\n",
+ fullpath, strerror(ret));
+ goto done;
+ } else if (ret == 1) {
+ // is_symlink_helper determined that the path is a symlink.
+ ret = unlink_helper(dirfd, name, 0);
+ if (ret) {
+ fprintf(LOGFILE, "failed to unlink symlink %s: %s\n",
+ fullpath, strerror(ret));
+ }
+ goto done;
+ }
+
+ // Open the file. We use O_NOFOLLOW here to ensure that we if a symlink was
+ // swapped in by an attacker, we will fail to follow it rather than deleting
+ // something we potentially should not.
fd = open_helper(dirfd, name);
if (fd == -EACCES) {
ret = chmod_helper(dirfd, name, 0700);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c58a6d53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index b16c6bc..ab29543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -750,6 +750,100 @@ void test_run_container() {
check_pid_file(cgroups_pids[1], child);
}
+static void mkdir_or_die(const char *path) {
+ if (mkdir(path, 0777) < 0) {
+ int err = errno;
+ printf("mkdir(%s) failed: %s\n", path, strerror(err));
+ exit(1);
+ }
+}
+
+static void touch_or_die(const char *path) {
+ FILE* f = fopen(path, "w");
+ if (!f) {
+ int err = errno;
+ printf("fopen(%s, w) failed: %s\n", path, strerror(err));
+ exit(1);
+ }
+ if (fclose(f) < 0) {
+ int err = errno;
+ printf("fclose(%s) failed: %s\n", path, strerror(err));
+ exit(1);
+ }
+}
+
+static void symlink_or_die(const char *old, const char *new) {
+ if (symlink(old, new) < 0) {
+ int err = errno;
+ printf("symlink(%s, %s) failed: %s\n", old, new, strerror(err));
+ exit(1);
+ }
+}
+
+static void expect_type(const char *path, int mode) {
+ struct stat st_buf;
+
+ if (stat(path, &st_buf) < 0) {
+ int err = errno;
+ if (err == ENOENT) {
+ if (mode == 0) {
+ return;
+ }
+ printf("expect_type(%s): stat failed unexpectedly: %s\n",
+ path, strerror(err));
+ exit(1);
+ }
+ }
+ if (mode == 0) {
+ printf("expect_type(%s): we expected the file to be gone, but it "
+ "existed.\n", path);
+ exit(1);
+ }
+ if (!(st_buf.st_mode & mode)) {
+ printf("expect_type(%s): the file existed, but it had mode 0%4o, "
+ "which didn't have bit 0%4o\n", path, st_buf.st_mode, mode);
+ exit(1);
+ }
+}
+
+int recursive_unlink_children(const char *name);
+
+void test_recursive_unlink_children() {
+ int ret;
+
+ mkdir_or_die(TEST_ROOT "/unlinkRoot");
+ mkdir_or_die(TEST_ROOT "/unlinkRoot/a");
+ touch_or_die(TEST_ROOT "/unlinkRoot/b");
+ mkdir_or_die(TEST_ROOT "/unlinkRoot/c");
+ touch_or_die(TEST_ROOT "/unlinkRoot/c/d");
+ touch_or_die(TEST_ROOT "/external");
+ symlink_or_die(TEST_ROOT "/external",
+ TEST_ROOT "/unlinkRoot/c/external");
+ ret = recursive_unlink_children(TEST_ROOT "/unlinkRoot");
+ if (ret != 0) {
+ printf("recursive_unlink_children(%s) failed: error %d\n",
+ TEST_ROOT "/unlinkRoot", ret);
+ exit(1);
+ }
+ // unlinkRoot should still exist.
+ expect_type(TEST_ROOT "/unlinkRoot", S_IFDIR);
+ // Other files under unlinkRoot should have been deleted.
+ expect_type(TEST_ROOT "/unlinkRoot/a", 0);
+ expect_type(TEST_ROOT "/unlinkRoot/b", 0);
+ expect_type(TEST_ROOT "/unlinkRoot/c", 0);
+ // We shouldn't have followed the symlink.
+ expect_type(TEST_ROOT "/external", S_IFREG);
+ // Clean up.
+ if (rmdir(TEST_ROOT "/unlinkRoot") < 0) {
+ int err = errno;
+ printf("failed to rmdir " TEST_ROOT "/unlinkRoot: %s\n", strerror(err));
+ }
+ if (unlink(TEST_ROOT "/external") < 0) {
+ int err = errno;
+ printf("failed to unlink " TEST_ROOT "/external: %s\n", strerror(err));
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -775,7 +869,7 @@ int main(int argc, char **argv) {
if (mkdirs(TEST_ROOT "/logs/userlogs", 0755) != 0) {
exit(1);
}
-
+
if (write_config_file(TEST_ROOT "/test.cfg", 1) != 0) {
exit(1);
}
@@ -804,6 +898,9 @@ int main(int argc, char **argv) {
printf("\nStarting tests\n");
+ printf("\nTesting recursive_unlink_children()\n");
+ test_recursive_unlink_children();
+
printf("\nTesting resolve_config_path()\n");
test_resolve_config_path();
[47/50] [abbrv] hadoop git commit: HADOOP-12872. Fix formatting in
ServiceLevelAuth.md. Contributed by Brahma Reddy Battula.
Posted by ar...@apache.org.
HADOOP-12872. Fix formatting in ServiceLevelAuth.md. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a9f00af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a9f00af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a9f00af
Branch: refs/heads/HDFS-1312
Commit: 0a9f00af5e11913935ebf426d9da74278dd011af
Parents: 1c3136c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 3 23:32:30 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 3 23:32:30 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/ServiceLevelAuth.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a9f00af/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d8c2b59..78ac5d7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1857,6 +1857,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12871. Fix dead link to NativeLibraries.html in CommandsManual.md.
(Brahma Reddy Battula via aajisaka)
+ HADOOP-12872. Fix formatting in ServiceLevelAuth.md.
+ (Brahma Reddy Battula via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a9f00af/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
index e0017d4..4e6acac 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
@@ -115,7 +115,7 @@ Of course, one can use the `security.refresh.policy.protocol.acl` property in `$
It is possible to specify a blocked list of hosts. Only those machines which are in the hosts list, but not in the blocked hosts list will be granted access to the service. The property name is derived by suffixing with ".blocked".
- Example: The property name of blocked hosts list for `security.client.protocol.hosts>> will be <<<security.client.protocol.hosts.blocked`
+ Example: The property name of blocked hosts list for `security.client.protocol.hosts` will be `security.client.protocol.hosts.blocked`
If blocked hosts list is not defined for a service, the value of `security.service.authorization.default.hosts.blocked` is applied. If `security.service.authorization.default.hosts.blocked` is not defined, empty blocked hosts list is applied.
[05/50] [abbrv] hadoop git commit: HADOOP-12831.
LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0
(Mingliang Liu via gtcarrera9)
Posted by ar...@apache.org.
HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0 (Mingliang Liu via gtcarrera9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7545ce66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7545ce66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7545ce66
Branch: refs/heads/HDFS-1312
Commit: 7545ce6636066a05763744a817878e03ee87f456
Parents: 03cfb45
Author: Li Lu <gt...@apache.org>
Authored: Sat Feb 27 21:59:03 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Sat Feb 27 21:59:03 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/fs/ChecksumFileSystem.java | 4 ++++
.../hadoop/fs/TestChecksumFileSystem.java | 23 ++++++++++++++++++++
3 files changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b8ae8df..9f95221 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -780,6 +780,9 @@ Release 2.8.0 - UNRELEASED
(Larry McCay via cnauroth)
IMPROVEMENTS
+
+ HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum
+ set to 0 (Mingliang Liu via gtcarrera9)
HADOOP-12458. Retries is typoed to spell Retires in parts of
hadoop-yarn and hadoop-common
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 3b8ecea..c19be3d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.nio.channels.ClosedChannelException;
import java.util.Arrays;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +62,9 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
if (conf != null) {
bytesPerChecksum = conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY,
LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT);
+ Preconditions.checkState(bytesPerChecksum > 0,
+ "bytes per checksum should be positive but was %s",
+ bytesPerChecksum);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
index 0c24ad5..923d219 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
@@ -228,6 +228,29 @@ public class TestChecksumFileSystem {
}
+ @Test
+ public void testSetConf() {
+ Configuration conf = new Configuration();
+
+ conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 0);
+ try {
+ localFs.setConf(conf);
+ fail("Should have failed because zero bytes per checksum is invalid");
+ } catch (IllegalStateException ignored) {
+ }
+
+ conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, -1);
+ try {
+ localFs.setConf(conf);
+ fail("Should have failed because negative bytes per checksum is invalid");
+ } catch (IllegalStateException ignored) {
+ }
+
+ conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 512);
+ localFs.setConf(conf);
+
+ }
+
void verifyRename(Path srcPath, Path dstPath, boolean dstIsDir)
throws Exception {
localFs.delete(srcPath,true);
[46/50] [abbrv] hadoop git commit: HADOOP-12871. Fix dead link to
NativeLibraries.html in CommandsManual.md. Contributed by Brahma Reddy
Battula.
Posted by ar...@apache.org.
HADOOP-12871. Fix dead link to NativeLibraries.html in CommandsManual.md. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c3136cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c3136cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c3136cb
Branch: refs/heads/HDFS-1312
Commit: 1c3136cbd8ff5f4eaf9c2c0145975439d0249db3
Parents: 33a412e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 3 23:22:35 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 3 23:22:35 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/CommandsManual.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3136cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bd3595..d8c2b59 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1854,6 +1854,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12870. Fix typo admininistration in CommandsManual.md.
(John Zhuge via aajisaka)
+ HADOOP-12871. Fix dead link to NativeLibraries.html in CommandsManual.md.
+ (Brahma Reddy Battula via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3136cb/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index b029e05..7b1fca3 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -109,7 +109,7 @@ Usage: `hadoop checknative [-a] [-h] `
| `-a` | Check all libraries are available. |
| `-h` | print help |
-This command checks the availability of the Hadoop native code. See [\#NativeLibraries.html](#NativeLibraries.html) for more information. By default, this command only checks the availability of libhadoop.
+This command checks the availability of the Hadoop native code. See [Native Libaries](./NativeLibraries.html) for more information. By default, this command only checks the availability of libhadoop.
### `classpath`
[08/50] [abbrv] hadoop git commit: HADOOP-12846. Credential Provider
Recursive Dependencies. Contributed by Larry McCay.
Posted by ar...@apache.org.
HADOOP-12846. Credential Provider Recursive Dependencies. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7634d404
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7634d404
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7634d404
Branch: refs/heads/HDFS-1312
Commit: 7634d404b750eafa135a37fa275325d0398255fb
Parents: f969277
Author: Chris Nauroth <cn...@apache.org>
Authored: Sun Feb 28 11:22:55 2016 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Sun Feb 28 11:22:55 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/security/ProviderUtils.java | 107 +++++++++++++++++--
.../hadoop/fs/azure/SimpleKeyProvider.java | 5 +-
.../fs/azure/TestWasbUriAndConfiguration.java | 35 ++++++
4 files changed, 143 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7634d404/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9f95221..4739564 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1761,6 +1761,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12849. TestSymlinkLocalFSFileSystem fails intermittently.
(Mingliang Liu via cnauroth)
+ HADOOP-12846. Credential Provider Recursive Dependencies.
+ (Larry McCay via cnauroth)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7634d404/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
index b764506..ae08fba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
@@ -18,14 +18,34 @@
package org.apache.hadoop.security;
+import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider;
-public class ProviderUtils {
+/**
+ * Utility methods for both key and credential provider APIs.
+ *
+ */
+public final class ProviderUtils {
+ private static final Log LOG = LogFactory.getLog(ProviderUtils.class);
+
+ /**
+ * Hidden ctor to ensure that this utility class isn't
+ * instantiated explicitly.
+ */
+ private ProviderUtils() {
+ // hide ctor for checkstyle compliance
+ }
+
/**
* Convert a nested URI to decode the underlying path. The translation takes
* the authority and parses it into the underlying scheme and authority.
@@ -35,11 +55,15 @@ public class ProviderUtils {
* @return the unnested path
*/
public static Path unnestUri(URI nestedUri) {
- String[] parts = nestedUri.getAuthority().split("@", 2);
- StringBuilder result = new StringBuilder(parts[0]);
- result.append("://");
- if (parts.length == 2) {
- result.append(parts[1]);
+ StringBuilder result = new StringBuilder();
+ String authority = nestedUri.getAuthority();
+ if (authority != null) {
+ String[] parts = nestedUri.getAuthority().split("@", 2);
+ result.append(parts[0]);
+ result.append("://");
+ if (parts.length == 2) {
+ result.append(parts[1]);
+ }
}
result.append(nestedUri.getPath());
if (nestedUri.getQuery() != null) {
@@ -79,4 +103,75 @@ public class ProviderUtils {
"//file" + localFile.getSchemeSpecificPart(), localFile.getFragment());
}
+ /**
+ * There are certain integrations of the credential provider API in
+ * which a recursive dependency between the provider and the hadoop
+ * filesystem abstraction causes a problem. These integration points
+ * need to leverage this utility method to remove problematic provider
+ * types from the existing provider path within the configuration.
+ *
+ * @param config the existing configuration with provider path
+ * @param fileSystemClass the class which providers must be compatible
+ * @return Configuration clone with new provider path
+ */
+ public static Configuration excludeIncompatibleCredentialProviders(
+ Configuration config, Class<? extends FileSystem> fileSystemClass)
+ throws IOException {
+
+ String providerPath = config.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
+
+ if (providerPath == null) {
+ return config;
+ }
+ StringBuffer newProviderPath = new StringBuffer();
+ String[] providers = providerPath.split(",");
+ Path path = null;
+ for (String provider: providers) {
+ try {
+ path = unnestUri(new URI(provider));
+ Class<? extends FileSystem> clazz = null;
+ try {
+ String scheme = path.toUri().getScheme();
+ clazz = FileSystem.getFileSystemClass(scheme, config);
+ } catch (IOException ioe) {
+ // not all providers are filesystem based
+ // for instance user:/// will not be able to
+ // have a filesystem class associated with it.
+ if (newProviderPath.length() > 0) {
+ newProviderPath.append(",");
+ }
+ newProviderPath.append(provider);
+ }
+ if (clazz != null) {
+ if (fileSystemClass.isAssignableFrom(clazz)) {
+ LOG.debug("Filesystem based provider" +
+ " excluded from provider path due to recursive dependency: "
+ + provider);
+ } else {
+ if (newProviderPath.length() > 0) {
+ newProviderPath.append(",");
+ }
+ newProviderPath.append(provider);
+ }
+ }
+ } catch (URISyntaxException e) {
+ LOG.warn("Credential Provider URI is invalid." + provider);
+ }
+ }
+
+ String effectivePath = newProviderPath.toString();
+ if (effectivePath.equals(providerPath)) {
+ return config;
+ }
+
+ Configuration conf = new Configuration(config);
+ if (effectivePath.equals("")) {
+ conf.unset(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
+ } else {
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ effectivePath);
+ }
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7634d404/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
index 28e307e..5596f7e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
/**
* Key provider that simply returns the storage account key from the
@@ -41,7 +42,9 @@ public class SimpleKeyProvider implements KeyProvider {
throws KeyProviderException {
String key = null;
try {
- char[] keyChars = conf.getPassword(getStorageAccountKeyName(accountName));
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, NativeAzureFileSystem.class);
+ char[] keyChars = c.getPassword(getStorageAccountKeyName(accountName));
if (keyChars != null) {
key = new String(keyChars);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7634d404/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
index 06a5b62..cd9d1d4 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
@@ -461,4 +461,39 @@ public class TestWasbUriAndConfiguration {
FileSystem.closeAll();
}
}
+
+ @Test
+ public void testCredentialProviderPathExclusions() throws Exception {
+ String providerPath =
+ "user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," +
+ "jceks://hdfs@nn1.example.com/my/path/test.jceks";
+ Configuration config = new Configuration();
+ config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ providerPath);
+ String newPath = "user:///,jceks://hdfs@nn1.example.com/my/path/test.jceks";
+
+ excludeAndTestExpectations(config, newPath);
+ }
+
+ @Test
+ public void testExcludeAllProviderTypesFromConfig() throws Exception {
+ String providerPath =
+ "jceks://wasb/tmp/test.jceks," +
+ "jceks://wasb@/my/path/test.jceks";
+ Configuration config = new Configuration();
+ config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ providerPath);
+ String newPath = null;
+
+ excludeAndTestExpectations(config, newPath);
+ }
+
+ void excludeAndTestExpectations(Configuration config, String newPath)
+ throws Exception {
+ Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
+ config, NativeAzureFileSystem.class);
+ String effectivePath = conf.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null);
+ assertEquals(newPath, effectivePath);
+ }
}
[10/50] [abbrv] hadoop git commit: HDFS-9867. Missing block exception
should carry locatedBlocks information. Contributed by Mingliang Liu.
Posted by ar...@apache.org.
HDFS-9867. Missing block exception should carry locatedBlocks information. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/321a80c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/321a80c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/321a80c7
Branch: refs/heads/HDFS-1312
Commit: 321a80c759e887f52bb4f40c49328527f04560a1
Parents: 408f2c8
Author: Jing Zhao <ji...@apache.org>
Authored: Sun Feb 28 15:00:27 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Sun Feb 28 15:00:27 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java | 3 ++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/321a80c7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index d4174d8..f6547f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -605,7 +605,8 @@ public class DFSStripedInputStream extends DFSInputStream {
if (alignedStripe.missingChunksNum > parityBlkNum) {
clearFutures(futures.keySet());
throw new IOException(alignedStripe.missingChunksNum
- + " missing blocks, the stripe is: " + alignedStripe);
+ + " missing blocks, the stripe is: " + alignedStripe
+ + "; locatedBlocks is: " + locatedBlocks);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/321a80c7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 651f0a1..78d2721 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -449,6 +449,9 @@ Trunk (Unreleased)
HDFS-9866. BlockManager#chooseExcessReplicasStriped may weaken rack fault
tolerance. (jing9)
+ HDFS-9867. Missing block exception should carry locatedBlocks information.
+ (Mingliang Liu via jing9)
+
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and
[16/50] [abbrv] hadoop git commit: HADOOP-12850. pull shell code out
of hadoop-dist
Posted by ar...@apache.org.
HADOOP-12850. pull shell code out of hadoop-dist
Signed-off-by: Steve Loughran <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cb2f934
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cb2f934
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cb2f934
Branch: refs/heads/HDFS-1312
Commit: 1cb2f93451aa444fadd1b7ffa7825ba4a6ae74e3
Parents: c58a6d5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Sun Feb 28 16:51:51 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 29 08:43:17 2016 -0800
----------------------------------------------------------------------
dev-support/bin/dist-layout-stitching | 140 +++++++++++++++++++++++++++
dev-support/bin/dist-tar-stitching | 44 +++++++++
hadoop-dist/pom.xml | 148 ++++-------------------------
3 files changed, 202 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cb2f934/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
new file mode 100755
index 0000000..78533f9
--- /dev/null
+++ b/dev-support/bin/dist-layout-stitching
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# project.version
+VERSION=$1
+
+# project.build.directory
+BASEDIR=$2
+
+function run()
+{
+ declare res
+
+ echo "\$ ${*}"
+ "${@}"
+ res=$?
+ if [[ ${res} != 0 ]]; then
+ echo
+ echo "Failed!"
+ echo
+ exit "${res}"
+ fi
+}
+
+function findfileindir()
+{
+ declare file="$1"
+ declare dir="${2:-./share}"
+ declare count
+
+ count=$(find "${dir}" -iname "${file}" | wc -l)
+
+ #shellcheck disable=SC2086
+ echo ${count}
+}
+
+function copyifnotexists()
+{
+ declare src="$1"
+ declare dest="$2"
+
+ declare srcname
+ declare destdir
+
+ declare child
+ declare childpath
+
+ if [[ -f "${src}" ]]; then
+ srcname=${src##*/}
+ if [[ "${srcname}" != *.jar ||
+ $(findfileindir "${srcname}") -eq "0" ]]; then
+ destdir=$(dirname "${dest}")
+ mkdir -p "${destdir}"
+ cp -p "${src}" "${dest}"
+ fi
+ else
+ for childpath in "${src}"/*; do
+ child="${childpath##*/}"
+ if [[ "${child}" == "doc" ||
+ "${child}" == "webapps" ]]; then
+ mkdir -p "${dest}/${child}"
+ cp -r "${src}/${child}"/* "${dest}/${child}"
+ continue;
+ fi
+ copyifnotexists "${src}/${child}" "${dest}/${child}"
+ done
+ fi
+}
+
+#Copy all contents as is except the lib.
+#for libs check for existence in share directory, if not exist then only copy.
+function copy()
+{
+ declare src="$1"
+ declare dest="$2"
+
+ declare child
+ declare childpath
+
+ if [[ -d "${src}" ]]; then
+ for childpath in "${src}"/*; do
+ child="${childpath##*/}"
+
+ if [[ "${child}" == "share" ]]; then
+ copyifnotexists "${src}/${child}" "${dest}/${child}"
+ else
+ if [[ -d "${src}/${child}" ]]; then
+ mkdir -p "${dest}/${child}"
+ cp -pr "${src}/${child}"/* "${dest}/${child}"
+ else
+ cp -pr "${src}/${child}" "${dest}/${child}"
+ fi
+ fi
+ done
+ fi
+}
+
+# shellcheck disable=SC2164
+ROOT=$(cd "${BASEDIR}"/../..;pwd)
+echo
+echo "Current directory $(pwd)"
+echo
+run rm -rf "hadoop-${VERSION}"
+run mkdir "hadoop-${VERSION}"
+run cd "hadoop-${VERSION}"
+run cp -p "${ROOT}/LICENSE.txt" .
+run cp -p "${ROOT}/NOTICE.txt" .
+run cp -p "${ROOT}/README.txt" .
+
+# Copy hadoop-common first so that it have always have all dependencies.
+# Remaining projects will copy only libraries which are not present already in 'share' directory.
+run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
+run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" .
+run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
+run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
+run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" .
+run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
+run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
+
+#copy httpfs and kms as is
+run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
+run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
+
+echo
+echo "Hadoop dist layout available at: ${BASEDIR}/hadoop-${VERSION}"
+echo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cb2f934/dev-support/bin/dist-tar-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-tar-stitching b/dev-support/bin/dist-tar-stitching
new file mode 100755
index 0000000..20028b4
--- /dev/null
+++ b/dev-support/bin/dist-tar-stitching
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# project.version
+VERSION=$1
+
+# project.build.directory
+BASEDIR=$2
+
+function run()
+{
+ declare res
+
+ echo "\$ ${*}"
+ "${@}"
+ res=$?
+ if [[ ${res} != 0 ]]; then
+ echo
+ echo "Failed!"
+ echo
+ exit "${res}"
+ fi
+}
+
+
+run tar cf "hadoop-${VERSION}.tar" "hadoop-${VERSION}"
+run gzip -f "hadoop-${VERSION}.tar"
+echo
+echo "Hadoop dist tar available at: ${BASEDIR}/hadoop-${VERSION}.tar.gz"
+echo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cb2f934/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 9d37599..ce1bd92 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -83,151 +83,39 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
<id>dist</id>
<phase>prepare-package</phase>
<goals>
- <goal>run</goal>
+ <goal>exec</goal>
</goals>
<configuration>
- <target>
- <echo file="${project.build.directory}/dist-layout-stitching.sh">
- run() {
- echo "\$ ${@}"
- "${@}"
- res=$?
- if [ $res != 0 ]; then
- echo
- echo "Failed!"
- echo
- exit $res
- fi
- }
-
- findFileInDir(){
- local file="$1";
- local dir="${2:-./share}";
- local count=$(find "$dir" -iname "$file"|wc -l)
- echo "$count";
- }
-
- copyIfNotExists(){
- local src="$1"
- local srcName=$(basename "$src")
- local dest="$2";
- if [ -f "$src" ]; then
- if [[ "$srcName" != *.jar ]] || [ $(findFileInDir "$srcName") -eq "0" ]; then
- local destDir=$(dirname "$dest")
- mkdir -p "$destDir"
- cp "$src" "$dest"
- fi
- else
- for childPath in "$src"/* ;
- do
- child=$(basename "$childPath");
- if [ "$child" == "doc" ] || [ "$child" == "webapps" ]; then
- mkdir -p "$dest"/"$child"
- cp -r "$src"/"$child"/* "$dest"/"$child"
- continue;
- fi
- copyIfNotExists "$src"/"$child" "$dest"/"$child"
- done
- fi
- }
-
- #Copy all contents as is except the lib.
- #for libs check for existence in share directory, if not exist then only copy.
- copy(){
- local src="$1";
- local dest="$2";
- if [ -d "$src" ]; then
- for childPath in "$src"/* ;
- do
- child=$(basename "$childPath");
- if [ "$child" == "share" ]; then
- copyIfNotExists "$src"/"$child" "$dest"/"$child"
- else
- if [ -d "$src"/"$child" ]; then
- mkdir -p "$dest"/"$child"
- cp -r "$src"/"$child"/* "$dest"/"$child"
- else
- cp -r "$src"/"$child" "$dest"/"$child"
- fi
- fi
- done
- fi
- }
-
- # Shellcheck SC2086
- ROOT=$(cd "${project.build.directory}"/../..;pwd)
- echo
- echo "Current directory $(pwd)"
- echo
- run rm -rf hadoop-${project.version}
- run mkdir hadoop-${project.version}
- run cd hadoop-${project.version}
- run cp "$ROOT"/LICENSE.txt .
- run cp "$ROOT"/NOTICE.txt .
- run cp "$ROOT"/README.txt .
-
- # Copy hadoop-common first so that it have always have all dependencies.
- # Remaining projects will copy only libraries which are not present already in 'share' directory.
- run copy "$ROOT"/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version} .
- run copy "$ROOT"/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${project.version} .
- run copy "$ROOT"/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version} .
- run copy "$ROOT"/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version} .
- run copy "$ROOT"/hadoop-yarn-project/target/hadoop-yarn-project-${project.version} .
- run copy "$ROOT"/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version} .
- run copy "$ROOT"/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version} .
-
- #copy httpfs and kms as is
- run cp -r "$ROOT"/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
- run cp -r "$ROOT"/hadoop-common-project/hadoop-kms/target/hadoop-kms-${project.version}/* .
-
- echo
- echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
- echo
- </echo>
- <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-layout-stitching.sh"/>
- </exec>
- </target>
+ <executable>${basedir}/../dev-support/bin/dist-layout-stitching</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>${project.version}</argument>
+ <argument>${project.build.directory}</argument>
+ </arguments>
</configuration>
</execution>
<execution>
<id>tar</id>
<phase>package</phase>
<goals>
- <goal>run</goal>
+ <goal>exec</goal>
</goals>
<configuration>
- <target if="tar">
- <echo file="${project.build.directory}/dist-tar-stitching.sh">
- run() {
- echo "\$ ${@}"
- "${@}"
- res=$?
- if [ $res != 0 ]; then
- echo
- echo "Failed!"
- echo
- exit $res
- fi
- }
-
- run tar cf hadoop-${project.version}.tar hadoop-${project.version}
- run gzip -f hadoop-${project.version}.tar
- echo
- echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
- echo
- </echo>
- <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
- <arg line="./dist-tar-stitching.sh"/>
- </exec>
- </target>
+ <executable>${basedir}/../dev-support/bin/dist-tar-stitching</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>${project.version}</argument>
+ <argument>${project.build.directory}</argument>
+ </arguments>
</configuration>
</execution>
</executions>
[30/50] [abbrv] hadoop git commit: YARN-4671. There is no need to
acquire CS lock when completing a container. Contributed by Meng Ding
Posted by ar...@apache.org.
YARN-4671. There is no need to acquire CS lock when completing a container. Contributed by Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c465df9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c465df9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c465df9
Branch: refs/heads/HDFS-1312
Commit: 5c465df90414d43250d09084748ab2d41af44eea
Parents: 2c8496e
Author: Jian He <ji...@apache.org>
Authored: Tue Mar 1 13:14:12 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Mar 1 13:14:12 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../scheduler/capacity/CapacityScheduler.java | 8 ++---
.../capacity/TestCapacityScheduler.java | 35 ++++++++++++++++----
3 files changed, 35 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c465df9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e6e7af3..4934e89 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1480,6 +1480,9 @@ Release 2.8.0 - UNRELEASED
YARN-4748. ApplicationHistoryManagerOnTimelineStore should not
swallow exceptions on generateApplicationReport. (Li Lu via jianhe)
+ YARN-4671. There is no need to acquire CS lock when completing a container.
+ (Meng Ding via jianhe)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c465df9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index acc2782..6a1091d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -228,7 +228,7 @@ public class CapacityScheduler extends
private AsyncScheduleThread asyncSchedulerThread;
private RMNodeLabelsManager labelManager;
private SchedulerHealth schedulerHealth = new SchedulerHealth();
- long lastNodeUpdateTime;
+ volatile long lastNodeUpdateTime;
/**
* EXPERT
@@ -928,7 +928,6 @@ public class CapacityScheduler extends
}
@Override
- // Note: when AM asks to release container, we will acquire scheduler lock
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
@@ -1546,9 +1545,8 @@ public class CapacityScheduler extends
}
}
- @Lock(CapacityScheduler.class)
@Override
- protected synchronized void completedContainerInternal(
+ protected void completedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
@@ -1957,7 +1955,7 @@ public class CapacityScheduler extends
return this.schedulerHealth;
}
- private synchronized void setLastNodeUpdateTime(long time) {
+ private void setLastNodeUpdateTime(long time) {
this.lastNodeUpdateTime = time;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c465df9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 78ed0a6..c8c97e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1078,9 +1078,33 @@ public class TestCapacityScheduler {
RegisterApplicationMasterRequest.newInstance("localhost", 12345, "");
client.registerApplicationMaster(request);
+ // Allocate a container
+ List<ResourceRequest> asks = Collections.singletonList(
+ ResourceRequest.newInstance(
+ Priority.newInstance(1), "*", Resources.createResource(2 * GB), 1));
+ AllocateRequest allocateRequest =
+ AllocateRequest.newInstance(0, 0.0f, asks, null, null);
+ client.allocate(allocateRequest);
+
+ // Make sure the container is allocated in RM
+ nm1.nodeHeartbeat(true);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(applicationAttemptId, 2);
+ Assert.assertTrue(rm.waitForState(nm1, containerId2,
+ RMContainerState.ALLOCATED, 10 * 1000));
+
+ // Acquire the container
+ allocateRequest = AllocateRequest.newInstance(1, 0.0f, null, null, null);
+ client.allocate(allocateRequest);
+
+ // Launch the container
+ final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ RMContainer rmContainer = cs.getRMContainer(containerId2);
+ rmContainer.handle(
+ new RMContainerEvent(containerId2, RMContainerEventType.LAUNCHED));
+
// grab the scheduler lock from another thread
// and verify an allocate call in this thread doesn't block on it
- final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
final CyclicBarrier barrier = new CyclicBarrier(2);
Thread otherThread = new Thread(new Runnable() {
@Override
@@ -1089,9 +1113,7 @@ public class TestCapacityScheduler {
try {
barrier.await();
barrier.await();
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (BrokenBarrierException e) {
+ } catch (InterruptedException | BrokenBarrierException e) {
e.printStackTrace();
}
}
@@ -1099,8 +1121,9 @@ public class TestCapacityScheduler {
});
otherThread.start();
barrier.await();
- AllocateRequest allocateRequest =
- AllocateRequest.newInstance(0, 0.0f, null, null, null);
+ List<ContainerId> release = Collections.singletonList(containerId2);
+ allocateRequest =
+ AllocateRequest.newInstance(2, 0.0f, null, release, null);
client.allocate(allocateRequest);
barrier.await();
otherThread.join();
[36/50] [abbrv] hadoop git commit: HDFS-9851. NameNode throws NPE
when setPermission is called on a path that does not exist. Contributed by
Brahma Reddy Battula.
Posted by ar...@apache.org.
HDFS-9851. NameNode throws NPE when setPermission is called on a path that does not exist. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27e0681f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27e0681f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27e0681f
Branch: refs/heads/HDFS-1312
Commit: 27e0681f28ee896ada163bbbc08fd44d113e7d15
Parents: e2ddf82
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Mar 2 14:43:03 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Mar 2 14:43:03 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 3 ++-
.../hadoop/hdfs/server/namenode/FSDirectory.java | 6 +++++-
.../test/java/org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../org/apache/hadoop/security/TestPermission.java | 13 +++++++++++++
5 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e0681f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a87ffe0..8be05bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2974,6 +2974,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9766. TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently.
(Xiao Chen via aajisaka)
+ HDFS-9851. NameNode throws NPE when setPermission is called on a path that
+ does not exist. (Brahma Reddy Battula via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e0681f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 24d3360..d27cec5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.security.AccessControlException;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
@@ -388,7 +389,7 @@ class FSDirXAttrOp {
private static void checkXAttrChangeAccess(
FSDirectory fsd, INodesInPath iip, XAttr xAttr,
FSPermissionChecker pc)
- throws AccessControlException {
+ throws AccessControlException, FileNotFoundException {
if (fsd.isPermissionEnabled() && xAttr.getNameSpace() == XAttr.NameSpace
.USER) {
final INode inode = iip.getLastINode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e0681f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index a00591f..51b7747 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1547,7 +1547,11 @@ public class FSDirectory implements Closeable {
}
void checkOwner(FSPermissionChecker pc, INodesInPath iip)
- throws AccessControlException {
+ throws AccessControlException, FileNotFoundException {
+ if (iip.getLastINode() == null) {
+ throw new FileNotFoundException(
+ "Directory/File does not exist " + iip.getPath());
+ }
checkPermission(pc, iip, true, null, null, null, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e0681f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 0753ab8..41cd5c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -3344,7 +3344,7 @@ public class TestDFSShell {
fs.createSnapshot(reserved, "snap");
fail("Can't create snapshot on /.reserved");
} catch (FileNotFoundException e) {
- assertTrue(e.getMessage().contains("Directory does not exist"));
+ assertTrue(e.getMessage().contains("Directory/File does not exist"));
}
cluster.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27e0681f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
index 27af49c..5a17bbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java
@@ -51,6 +51,7 @@ public class TestPermission {
final private static Path CHILD_DIR2 = new Path(ROOT_PATH, "child2");
final private static Path CHILD_FILE1 = new Path(ROOT_PATH, "file1");
final private static Path CHILD_FILE2 = new Path(ROOT_PATH, "file2");
+ final private static Path CHILD_FILE3 = new Path(ROOT_PATH, "file3");
final private static int FILE_LEN = 100;
final private static Random RAN = new Random();
@@ -270,6 +271,18 @@ public class TestPermission {
final Path RENAME_PATH = new Path("/foo/bar");
userfs.mkdirs(RENAME_PATH);
assertTrue(canRename(userfs, RENAME_PATH, CHILD_DIR1));
+ // test permissions on files that do not exist
+ assertFalse(userfs.exists(CHILD_FILE3));
+ try {
+ userfs.setOwner(CHILD_FILE3, "foo", "bar");
+ fail("setOwner should fail for non-exist file");
+ } catch (java.io.FileNotFoundException ignored) {
+ }
+ try {
+ userfs.setPermission(CHILD_FILE3, new FsPermission((short) 0777));
+ fail("setPermission should fail for non-exist file");
+ } catch (java.io.FileNotFoundException ignored) {
+ }
} finally {
cluster.shutdown();
}
[18/50] [abbrv] hadoop git commit: HADOOP-12813. Migrate TestRPC and
related codes to rebase on ProtobufRpcEngine. Contributed by Kai Zheng.
Posted by ar...@apache.org.
HADOOP-12813. Migrate TestRPC and related codes to rebase on ProtobufRpcEngine. Contributed by Kai Zheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fa54d45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fa54d45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fa54d45
Branch: refs/heads/HDFS-1312
Commit: 0fa54d45b1cf8a29f089f64d24f35bd221b4803f
Parents: 1cb2f93
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Feb 29 11:41:00 2016 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Feb 29 11:41:00 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 4 +-
.../hadoop/ipc/TestMultipleProtocolServer.java | 14 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 137 +--
.../java/org/apache/hadoop/ipc/TestRPC.java | 1013 ++++++++----------
.../hadoop/ipc/TestRPCServerShutdown.java | 106 ++
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 295 +++++
.../hadoop-common/src/test/proto/test.proto | 33 +
.../src/test/proto/test_rpc_service.proto | 7 +
.../server/nodemanager/TestNMAuditLogger.java | 40 +-
.../resourcemanager/TestRMAuditLogger.java | 44 +-
11 files changed, 987 insertions(+), 709 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index eb33464..f444b71 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1763,6 +1763,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12846. Credential Provider Recursive Dependencies.
(Larry McCay via cnauroth)
+ HADOOP-12813. Migrate TestRPC and related codes to rebase on
+ ProtobufRpcEngine. (Kai Zheng via wheat9)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index 6400e87..eb7b949 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -34,8 +34,6 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.TestProtoBufRpc.PBServerImpl;
-import org.apache.hadoop.ipc.TestProtoBufRpc.TestRpcService;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
@@ -54,7 +52,7 @@ import com.google.protobuf.BlockingService;
* Benchmark for protobuf RPC.
* Run with --help option for usage.
*/
-public class RPCCallBenchmark implements Tool {
+public class RPCCallBenchmark extends TestRpcBase implements Tool {
private Configuration conf;
private AtomicLong callCount = new AtomicLong(0);
private static ThreadMXBean threadBean =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 29a293f..8b419e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -23,8 +23,6 @@ import java.net.InetSocketAddress;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.TestProtoBufRpc.PBServerImpl;
-import org.apache.hadoop.ipc.TestProtoBufRpc.TestRpcService;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
import org.apache.hadoop.net.NetUtils;
import org.junit.Before;
@@ -32,8 +30,7 @@ import org.junit.After;
import org.junit.Test;
import com.google.protobuf.BlockingService;
-public class TestMultipleProtocolServer {
- private static final String ADDRESS = "0.0.0.0";
+public class TestMultipleProtocolServer extends TestRpcBase {
private static InetSocketAddress addr;
private static RPC.Server server;
@@ -64,13 +61,12 @@ public class TestMultipleProtocolServer {
public static final long versionID = 0L;
void hello() throws IOException;
}
+
interface Bar extends Mixin {
public static final long versionID = 0L;
int echo(int i) throws IOException;
}
-
-
class Foo0Impl implements Foo0 {
@Override
@@ -185,8 +181,7 @@ public class TestMultipleProtocolServer {
// Add Protobuf server
// Create server side implementation
- PBServerImpl pbServerImpl =
- new PBServerImpl();
+ PBServerImpl pbServerImpl = new PBServerImpl();
BlockingService service = TestProtobufRpcProto
.newReflectiveBlockingService(pbServerImpl);
server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
@@ -241,8 +236,7 @@ public class TestMultipleProtocolServer {
FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
foo.ping();
}
-
-
+
/**
* getProtocolVersion of an unimplemented version should return highest version
* Similarly getProtocolSignature should work.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
index d9c9d6c..41ae910 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
@@ -17,15 +17,9 @@
*/
package org.apache.hadoop.ipc;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -36,76 +30,37 @@ import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EmptyRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EmptyResponseProto;
-import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpc2Proto;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.Test;
import org.junit.Before;
-import org.junit.After;
+import org.junit.Test;
+import java.io.IOException;
+import java.net.URISyntaxException;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Test for testing protocol buffer based RPC mechanism.
* This test depends on test.proto definition of types in src/test/proto
* and protobuf service definition from src/test/test_rpc_service.proto
*/
-public class TestProtoBufRpc {
- public final static String ADDRESS = "0.0.0.0";
- public final static int PORT = 0;
- private static InetSocketAddress addr;
- private static Configuration conf;
+public class TestProtoBufRpc extends TestRpcBase {
private static RPC.Server server;
private final static int SLEEP_DURATION = 1000;
- @ProtocolInfo(protocolName = "testProto", protocolVersion = 1)
- public interface TestRpcService
- extends TestProtobufRpcProto.BlockingInterface {
- }
-
@ProtocolInfo(protocolName = "testProto2", protocolVersion = 1)
public interface TestRpcService2 extends
TestProtobufRpc2Proto.BlockingInterface {
}
- public static class PBServerImpl implements TestRpcService {
-
- @Override
- public EmptyResponseProto ping(RpcController unused,
- EmptyRequestProto request) throws ServiceException {
- // Ensure clientId is received
- byte[] clientId = Server.getClientId();
- Assert.assertNotNull(Server.getClientId());
- Assert.assertEquals(16, clientId.length);
- return EmptyResponseProto.newBuilder().build();
- }
-
- @Override
- public EchoResponseProto echo(RpcController unused, EchoRequestProto request)
- throws ServiceException {
- return EchoResponseProto.newBuilder().setMessage(request.getMessage())
- .build();
- }
-
- @Override
- public EmptyResponseProto error(RpcController unused,
- EmptyRequestProto request) throws ServiceException {
- throw new ServiceException("error", new RpcServerException("error"));
- }
-
- @Override
- public EmptyResponseProto error2(RpcController unused,
- EmptyRequestProto request) throws ServiceException {
- throw new ServiceException("error", new URISyntaxException("",
- "testException"));
- }
- }
-
public static class PBServer2Impl implements TestRpcService2 {
@Override
@@ -133,12 +88,13 @@ public class TestProtoBufRpc {
}
@Before
- public void setUp() throws IOException { // Setup server for both protocols
+ public void setUp() throws IOException { // Setup server for both protocols
conf = new Configuration();
conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, 1024);
conf.setBoolean(CommonConfigurationKeys.IPC_SERVER_LOG_SLOW_RPC, true);
// Set RPC engine to protobuf RPC engine
RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class);
+ RPC.setProtocolEngine(conf, TestRpcService2.class, ProtobufRpcEngine.class);
// Create server side implementation
PBServerImpl serverImpl = new PBServerImpl();
@@ -149,12 +105,12 @@ public class TestProtoBufRpc {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(ADDRESS).setPort(PORT).build();
addr = NetUtils.getConnectAddress(server);
-
+
// now the second protocol
PBServer2Impl server2Impl = new PBServer2Impl();
BlockingService service2 = TestProtobufRpc2Proto
.newReflectiveBlockingService(server2Impl);
-
+
server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService2.class,
service2);
server.start();
@@ -166,31 +122,20 @@ public class TestProtoBufRpc {
server.stop();
}
- private static TestRpcService getClient() throws IOException {
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class);
- return RPC.getProxy(TestRpcService.class, 0, addr, conf);
- }
-
- private static TestRpcService2 getClient2() throws IOException {
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(conf, TestRpcService2.class,
- ProtobufRpcEngine.class);
- return RPC.getProxy(TestRpcService2.class, 0, addr,
- conf);
+ private TestRpcService2 getClient2() throws IOException {
+ return RPC.getProxy(TestRpcService2.class, 0, addr, conf);
}
@Test (timeout=5000)
public void testProtoBufRpc() throws Exception {
- TestRpcService client = getClient();
+ TestRpcService client = getClient(addr, conf);
testProtoBufRpc(client);
}
// separated test out so that other tests can call it.
public static void testProtoBufRpc(TestRpcService client) throws Exception {
// Test ping method
- EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build();
- client.ping(null, emptyRequest);
+ client.ping(null, newEmptyRequest());
// Test echo method
EchoRequestProto echoRequest = EchoRequestProto.newBuilder()
@@ -200,7 +145,7 @@ public class TestProtoBufRpc {
// Test error method - error should be thrown as RemoteException
try {
- client.error(null, emptyRequest);
+ client.error(null, newEmptyRequest());
Assert.fail("Expected exception is not thrown");
} catch (ServiceException e) {
RemoteException re = (RemoteException)e.getCause();
@@ -217,13 +162,11 @@ public class TestProtoBufRpc {
TestRpcService2 client = getClient2();
// Test ping method
- EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build();
- client.ping2(null, emptyRequest);
+ client.ping2(null, newEmptyRequest());
// Test echo method
- EchoRequestProto echoRequest = EchoRequestProto.newBuilder()
- .setMessage("hello").build();
- EchoResponseProto echoResponse = client.echo2(null, echoRequest);
+ EchoResponseProto echoResponse = client.echo2(null,
+ newEchoRequest("hello"));
Assert.assertEquals(echoResponse.getMessage(), "hello");
// Ensure RPC metrics are updated
@@ -238,11 +181,10 @@ public class TestProtoBufRpc {
@Test (timeout=5000)
public void testProtoBufRandomException() throws Exception {
- TestRpcService client = getClient();
- EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build();
+ TestRpcService client = getClient(addr, conf);
try {
- client.error2(null, emptyRequest);
+ client.error2(null, newEmptyRequest());
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof RemoteException);
RemoteException re = (RemoteException) se.getCause();
@@ -258,17 +200,14 @@ public class TestProtoBufRpc {
public void testExtraLongRpc() throws Exception {
TestRpcService2 client = getClient2();
final String shortString = StringUtils.repeat("X", 4);
- EchoRequestProto echoRequest = EchoRequestProto.newBuilder()
- .setMessage(shortString).build();
// short message goes through
- EchoResponseProto echoResponse = client.echo2(null, echoRequest);
+ EchoResponseProto echoResponse = client.echo2(null,
+ newEchoRequest(shortString));
Assert.assertEquals(shortString, echoResponse.getMessage());
final String longString = StringUtils.repeat("X", 4096);
- echoRequest = EchoRequestProto.newBuilder()
- .setMessage(longString).build();
try {
- echoResponse = client.echo2(null, echoRequest);
+ client.echo2(null, newEchoRequest(longString));
Assert.fail("expected extra-long RPC to fail");
} catch (ServiceException se) {
// expected
@@ -281,8 +220,7 @@ public class TestProtoBufRpc {
// make 10 K fast calls
for (int x = 0; x < 10000; x++) {
try {
- EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build();
- client.ping2(null, emptyRequest);
+ client.ping2(null, newEmptyRequest());
} catch (Exception ex) {
throw ex;
}
@@ -294,10 +232,7 @@ public class TestProtoBufRpc {
long before = rpcMetrics.getRpcSlowCalls();
// make a really slow call. Sleep sleeps for 1000ms
- TestProtos.SleepRequestProto sleepRequest =
- TestProtos.SleepRequestProto.newBuilder()
- .setMilliSeconds(SLEEP_DURATION * 3).build();
- TestProtos.SleepResponseProto Response = client.sleep(null, sleepRequest);
+ client.sleep(null, newSleepRequest(SLEEP_DURATION * 3));
long after = rpcMetrics.getRpcSlowCalls();
// Ensure slow call is logged.
@@ -312,8 +247,7 @@ public class TestProtoBufRpc {
// make 10 K fast calls
for (int x = 0; x < 10000; x++) {
- EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build();
- client.ping2(null, emptyRequest);
+ client.ping2(null, newEmptyRequest());
}
// Ensure RPC metrics are updated
@@ -322,10 +256,7 @@ public class TestProtoBufRpc {
long before = rpcMetrics.getRpcSlowCalls();
// make a really slow call. Sleep sleeps for 1000ms
- TestProtos.SleepRequestProto sleepRequest =
- TestProtos.SleepRequestProto.newBuilder()
- .setMilliSeconds(SLEEP_DURATION).build();
- TestProtos.SleepResponseProto Response = client.sleep(null, sleepRequest);
+ client.sleep(null, newSleepRequest(SLEEP_DURATION));
long after = rpcMetrics.getRpcSlowCalls();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index e19ef34..99bfc61 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -18,59 +18,20 @@
package org.apache.hadoop.ipc;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
-import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-
-import java.io.Closeable;
-import java.io.InterruptedIOException;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-import java.lang.management.ThreadMXBean;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.net.ConnectException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.net.SocketFactory;
-
+import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.UTF8;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.Client.ConnectionId;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
import org.apache.hadoop.ipc.Server.Call;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -89,35 +50,59 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
-import com.google.protobuf.DescriptorProtos;
-import com.google.protobuf.DescriptorProtos.EnumDescriptorProto;
+import javax.net.SocketFactory;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.ConnectException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
/** Unit tests for RPC. */
@SuppressWarnings("deprecation")
-public class TestRPC {
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG =
- LogFactory.getLog(TestRPC.class);
-
- private static Configuration conf;
-
+public class TestRPC extends TestRpcBase {
+
+ public static final Log LOG = LogFactory.getLog(TestRPC.class);
+
@Before
- public void setupConf() {
- conf = new Configuration();
- conf.setClass("rpc.engine." + StoppedProtocol.class.getName(),
- StoppedRpcEngine.class, RpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
+ public void setup() {
+ setupConf();
}
int datasize = 1024*100;
int numThreads = 50;
public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
+ long versionID = 1L;
+
void ping() throws IOException;
- void slowPing(boolean shouldSlow) throws IOException;
void sleep(long delay) throws IOException, InterruptedException;
String echo(String value) throws IOException;
String[] echo(String[] value) throws IOException;
@@ -125,11 +110,6 @@ public class TestRPC {
int add(int v1, int v2) throws IOException;
int add(int[] values) throws IOException;
int error() throws IOException;
- void testServerGet() throws IOException;
- int[] exchange(int[] values) throws IOException;
-
- DescriptorProtos.EnumDescriptorProto exchangeProto(
- DescriptorProtos.EnumDescriptorProto arg);
}
public static class TestImpl implements TestProtocol {
@@ -139,36 +119,21 @@ public class TestRPC {
public long getProtocolVersion(String protocol, long clientVersion) {
return TestProtocol.versionID;
}
-
+
@Override
public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
- int hashcode) {
+ int hashcode) {
return new ProtocolSignature(TestProtocol.versionID, null);
}
-
- @Override
- public void ping() {}
@Override
- public synchronized void slowPing(boolean shouldSlow) {
- if (shouldSlow) {
- while (fastPingCounter < 2) {
- try {
- wait(); // slow response until two fast pings happened
- } catch (InterruptedException ignored) {}
- }
- fastPingCounter -= 2;
- } else {
- fastPingCounter++;
- notify();
- }
- }
+ public void ping() {}
@Override
public void sleep(long delay) throws InterruptedException {
Thread.sleep(delay);
}
-
+
@Override
public String echo(String value) throws IOException { return value; }
@@ -197,26 +162,6 @@ public class TestRPC {
public int error() throws IOException {
throw new IOException("bobo");
}
-
- @Override
- public void testServerGet() throws IOException {
- if (!(Server.get() instanceof RPC.Server)) {
- throw new IOException("Server.get() failed");
- }
- }
-
- @Override
- public int[] exchange(int[] values) {
- for (int i = 0; i < values.length; i++) {
- values[i] = i;
- }
- return values;
- }
-
- @Override
- public EnumDescriptorProto exchangeProto(EnumDescriptorProto arg) {
- return arg;
- }
}
//
@@ -224,9 +169,9 @@ public class TestRPC {
//
static class Transactions implements Runnable {
int datasize;
- TestProtocol proxy;
+ TestRpcService proxy;
- Transactions(TestProtocol proxy, int datasize) {
+ Transactions(TestRpcService proxy, int datasize) {
this.proxy = proxy;
this.datasize = datasize;
}
@@ -234,19 +179,33 @@ public class TestRPC {
// do two RPC that transfers data.
@Override
public void run() {
- int[] indata = new int[datasize];
- int[] outdata = null;
+ Integer[] indata = new Integer[datasize];
+ Arrays.fill(indata, 123);
+ TestProtos.ExchangeRequestProto exchangeRequest =
+ TestProtos.ExchangeRequestProto.newBuilder().addAllValues(
+ Arrays.asList(indata)).build();
+ Integer[] outdata = null;
+ TestProtos.ExchangeResponseProto exchangeResponse;
+
+ TestProtos.AddRequestProto addRequest =
+ TestProtos.AddRequestProto.newBuilder().setParam1(1)
+ .setParam2(2).build();
+ TestProtos.AddResponseProto addResponse;
+
int val = 0;
try {
- outdata = proxy.exchange(indata);
- val = proxy.add(1,2);
- } catch (IOException e) {
+ exchangeResponse = proxy.exchange(null, exchangeRequest);
+ outdata = new Integer[exchangeResponse.getValuesCount()];
+ outdata = exchangeResponse.getValuesList().toArray(outdata);
+ addResponse = proxy.add(null, addRequest);
+ val = addResponse.getResult();
+ } catch (ServiceException e) {
assertTrue("Exception from RPC exchange() " + e, false);
}
assertEquals(indata.length, outdata.length);
assertEquals(3, val);
for (int i = 0; i < outdata.length; i++) {
- assertEquals(outdata[i], i);
+ assertEquals(outdata[i].intValue(), i);
}
}
}
@@ -255,10 +214,10 @@ public class TestRPC {
// A class that does an RPC but does not read its response.
//
static class SlowRPC implements Runnable {
- private TestProtocol proxy;
+ private TestRpcService proxy;
private volatile boolean done;
-
- SlowRPC(TestProtocol proxy) {
+
+ SlowRPC(TestRpcService proxy) {
this.proxy = proxy;
done = false;
}
@@ -270,54 +229,62 @@ public class TestRPC {
@Override
public void run() {
try {
- proxy.slowPing(true); // this would hang until two fast pings happened
+ // this would hang until two fast pings happened
+ ping(true);
done = true;
- } catch (IOException e) {
+ } catch (ServiceException e) {
assertTrue("SlowRPC ping exception " + e, false);
}
}
+
+ void ping(boolean shouldSlow) throws ServiceException {
+ // this would hang until two fast pings happened
+ proxy.slowPing(null, newSlowPingRequest(shouldSlow));
+ }
}
-
+
/**
* A basic interface for testing client-side RPC resource cleanup.
*/
- private static interface StoppedProtocol {
+ private interface StoppedProtocol {
long versionID = 0;
- public void stop();
+ void stop();
}
-
+
/**
* A class used for testing cleanup of client side RPC resources.
*/
private static class StoppedRpcEngine implements RpcEngine {
@Override
- public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
- InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
- SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy
- ) throws IOException {
+ public <T> ProtocolProxy<T> getProxy(
+ Class<T> protocol, long clientVersion, InetSocketAddress addr,
+ UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout,
+ RetryPolicy connectionRetryPolicy) throws IOException {
return getProxy(protocol, clientVersion, addr, ticket, conf, factory,
- rpcTimeout, connectionRetryPolicy, null);
+ rpcTimeout, connectionRetryPolicy, null);
}
@SuppressWarnings("unchecked")
@Override
- public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
- InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
- SocketFactory factory, int rpcTimeout,
- RetryPolicy connectionRetryPolicy, AtomicBoolean fallbackToSimpleAuth
- ) throws IOException {
+ public <T> ProtocolProxy<T> getProxy(
+ Class<T> protocol, long clientVersion, InetSocketAddress addr,
+ UserGroupInformation ticket, Configuration conf, SocketFactory factory,
+ int rpcTimeout, RetryPolicy connectionRetryPolicy,
+ AtomicBoolean fallbackToSimpleAuth) throws IOException {
T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(),
- new Class[] { protocol }, new StoppedInvocationHandler());
+ new Class[] { protocol }, new StoppedInvocationHandler());
return new ProtocolProxy<T>(protocol, proxy, false);
}
@Override
- public org.apache.hadoop.ipc.RPC.Server getServer(Class<?> protocol,
- Object instance, String bindAddress, int port, int numHandlers,
- int numReaders, int queueSizePerHandler, boolean verbose, Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager,
+ public org.apache.hadoop.ipc.RPC.Server getServer(
+ Class<?> protocol, Object instance, String bindAddress, int port,
+ int numHandlers, int numReaders, int queueSizePerHandler,
+ boolean verbose, Configuration conf,
+ SecretManager<? extends TokenIdentifier> secretManager,
String portRangeConfig) throws IOException {
return null;
}
@@ -336,278 +303,265 @@ public class TestRPC {
*/
private static class StoppedInvocationHandler
implements InvocationHandler, Closeable {
-
+
private int closeCalled = 0;
@Override
public Object invoke(Object proxy, Method method, Object[] args)
throws Throwable {
- return null;
+ return null;
}
@Override
public void close() throws IOException {
closeCalled++;
}
-
+
public int getCloseCalled() {
return closeCalled;
}
-
+
}
-
+
@Test
public void testConfRpc() throws IOException {
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ Server server = newServerBuilder(conf)
.setNumHandlers(1).setVerbose(false).build();
+
// Just one handler
int confQ = conf.getInt(
- CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
- CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT);
+ CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
+ CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT);
assertEquals(confQ, server.getMaxQueueSize());
int confReaders = conf.getInt(
- CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY,
- CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_DEFAULT);
+ CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY,
+ CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_DEFAULT);
assertEquals(confReaders, server.getNumReaders());
- server.stop();
-
- server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+
+ server = newServerBuilder(conf)
.setNumHandlers(1).setnumReaders(3).setQueueSizePerHandler(200)
- .setVerbose(false).build();
-
+ .setVerbose(false).build();
+
assertEquals(3, server.getNumReaders());
assertEquals(200, server.getMaxQueueSize());
- server.stop();
}
@Test
- public void testProxyAddress() throws IOException {
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0).build();
- TestProtocol proxy = null;
-
- try {
- server.start();
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ public void testProxyAddress() throws Exception {
+ Server server = null;
+ TestRpcService proxy = null;
+ try {
+ server = setupTestServer(conf, -1);
// create a client
- proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
-
+ proxy = getClient(addr, conf);
assertEquals(addr, RPC.getServerAddress(proxy));
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, proxy);
}
}
@Test
- public void testSlowRpc() throws IOException {
+ public void testSlowRpc() throws IOException, ServiceException {
+ Server server;
+ TestRpcService proxy = null;
+
System.out.println("Testing Slow RPC");
// create a server with two handlers
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
-
- TestProtocol proxy = null;
-
+ server = setupTestServer(conf, 2);
+
try {
- server.start();
+ // create a client
+ proxy = getClient(addr, conf);
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ SlowRPC slowrpc = new SlowRPC(proxy);
+ Thread thread = new Thread(slowrpc, "SlowRPC");
+ thread.start(); // send a slow RPC, which won't return until two fast pings
+ assertTrue("Slow RPC should not have finished1.", !slowrpc.isDone());
- // create a client
- proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
+ slowrpc.ping(false); // first fast ping
- SlowRPC slowrpc = new SlowRPC(proxy);
- Thread thread = new Thread(slowrpc, "SlowRPC");
- thread.start(); // send a slow RPC, which won't return until two fast pings
- assertTrue("Slow RPC should not have finished1.", !slowrpc.isDone());
+ // verify that the first RPC is still stuck
+ assertTrue("Slow RPC should not have finished2.", !slowrpc.isDone());
- proxy.slowPing(false); // first fast ping
-
- // verify that the first RPC is still stuck
- assertTrue("Slow RPC should not have finished2.", !slowrpc.isDone());
+ slowrpc.ping(false); // second fast ping
- proxy.slowPing(false); // second fast ping
-
- // Now the slow ping should be able to be executed
- while (!slowrpc.isDone()) {
- System.out.println("Waiting for slow RPC to get done.");
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {}
- }
- } finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
+ // Now the slow ping should be able to be executed
+ while (!slowrpc.isDone()) {
+ System.out.println("Waiting for slow RPC to get done.");
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {}
}
+ } finally {
System.out.println("Down slow rpc testing");
+ stop(server, proxy);
}
}
-
+
@Test
- public void testCalls() throws IOException {
+ public void testCalls() throws Exception {
testCallsInternal(conf);
}
-
- private void testCallsInternal(Configuration conf) throws IOException {
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0).build();
- TestProtocol proxy = null;
- try {
- server.start();
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
- proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
-
- proxy.ping();
+ private void testCallsInternal(Configuration myConf) throws Exception {
+ Server server;
+ TestRpcService proxy = null;
- String stringResult = proxy.echo("foo");
- assertEquals(stringResult, "foo");
-
- stringResult = proxy.echo((String)null);
- assertEquals(stringResult, null);
-
- // Check rpcMetrics
- MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
- assertCounter("RpcProcessingTimeNumOps", 3L, rb);
- assertCounterGt("SentBytes", 0L, rb);
- assertCounterGt("ReceivedBytes", 0L, rb);
-
- // Number of calls to echo method should be 2
- rb = getMetrics(server.rpcDetailedMetrics.name());
- assertCounter("EchoNumOps", 2L, rb);
-
- // Number of calls to ping method should be 1
- assertCounter("PingNumOps", 1L, rb);
-
- String[] stringResults = proxy.echo(new String[]{"foo","bar"});
- assertTrue(Arrays.equals(stringResults, new String[]{"foo","bar"}));
-
- stringResults = proxy.echo((String[])null);
- assertTrue(Arrays.equals(stringResults, null));
-
- UTF8 utf8Result = (UTF8)proxy.echo(new UTF8("hello world"));
- assertEquals(new UTF8("hello world"), utf8Result );
-
- utf8Result = (UTF8)proxy.echo((UTF8)null);
- assertEquals(null, utf8Result);
+ server = setupTestServer(myConf, -1);
+ try {
+ proxy = getClient(addr, myConf);
- int intResult = proxy.add(1, 2);
- assertEquals(intResult, 3);
+ proxy.ping(null, newEmptyRequest());
- intResult = proxy.add(new int[] {1, 2});
- assertEquals(intResult, 3);
+ TestProtos.EchoResponseProto echoResp = proxy.echo(null,
+ newEchoRequest("foo"));
+ assertEquals(echoResp.getMessage(), "foo");
- // Test protobufs
- EnumDescriptorProto sendProto =
- EnumDescriptorProto.newBuilder().setName("test").build();
- EnumDescriptorProto retProto = proxy.exchangeProto(sendProto);
- assertEquals(sendProto, retProto);
- assertNotSame(sendProto, retProto);
+ echoResp = proxy.echo(null, newEchoRequest(""));
+ assertEquals(echoResp.getMessage(), "");
- boolean caught = false;
- try {
- proxy.error();
- } catch (IOException e) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Caught " + e);
- }
- caught = true;
- }
- assertTrue(caught);
- rb = getMetrics(server.rpcDetailedMetrics.name());
- assertCounter("IOExceptionNumOps", 1L, rb);
-
- proxy.testServerGet();
-
- // create multiple threads and make them do large data transfers
- System.out.println("Starting multi-threaded RPC test...");
- server.setSocketSendBufSize(1024);
- Thread threadId[] = new Thread[numThreads];
- for (int i = 0; i < numThreads; i++) {
- Transactions trans = new Transactions(proxy, datasize);
- threadId[i] = new Thread(trans, "TransactionThread-" + i);
- threadId[i].start();
- }
-
- // wait for all transactions to get over
- System.out.println("Waiting for all threads to finish RPCs...");
- for (int i = 0; i < numThreads; i++) {
+ // Check rpcMetrics
+ MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
+ assertCounter("RpcProcessingTimeNumOps", 3L, rb);
+ assertCounterGt("SentBytes", 0L, rb);
+ assertCounterGt("ReceivedBytes", 0L, rb);
+
+ // Number of calls to echo method should be 2
+ rb = getMetrics(server.rpcDetailedMetrics.name());
+ assertCounter("EchoNumOps", 2L, rb);
+
+ // Number of calls to ping method should be 1
+ assertCounter("PingNumOps", 1L, rb);
+
+ String[] strings = new String[] {"foo","bar"};
+ TestProtos.EchoRequestProto2 echoRequest2 =
+ TestProtos.EchoRequestProto2.newBuilder().addAllMessage(
+ Arrays.asList(strings)).build();
+ TestProtos.EchoResponseProto2 echoResponse2 =
+ proxy.echo2(null, echoRequest2);
+ assertTrue(Arrays.equals(echoResponse2.getMessageList().toArray(),
+ strings));
+
+ echoRequest2 = TestProtos.EchoRequestProto2.newBuilder()
+ .addAllMessage(Collections.<String>emptyList()).build();
+ echoResponse2 = proxy.echo2(null, echoRequest2);
+ assertTrue(Arrays.equals(echoResponse2.getMessageList().toArray(),
+ new String[]{}));
+
+ TestProtos.AddRequestProto addRequest =
+ TestProtos.AddRequestProto.newBuilder().setParam1(1)
+ .setParam2(2).build();
+ TestProtos.AddResponseProto addResponse =
+ proxy.add(null, addRequest);
+ assertEquals(addResponse.getResult(), 3);
+
+ Integer[] integers = new Integer[] {1, 2};
+ TestProtos.AddRequestProto2 addRequest2 =
+ TestProtos.AddRequestProto2.newBuilder().addAllParams(
+ Arrays.asList(integers)).build();
+ addResponse = proxy.add2(null, addRequest2);
+ assertEquals(addResponse.getResult(), 3);
+
+ boolean caught = false;
try {
- threadId[i].join();
- } catch (InterruptedException e) {
- i--; // retry
+ proxy.error(null, newEmptyRequest());
+ } catch (ServiceException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Caught " + e);
+ }
+ caught = true;
+ }
+ assertTrue(caught);
+ rb = getMetrics(server.rpcDetailedMetrics.name());
+ assertCounter("RpcServerExceptionNumOps", 1L, rb);
+
+ //proxy.testServerGet();
+
+ // create multiple threads and make them do large data transfers
+ System.out.println("Starting multi-threaded RPC test...");
+ server.setSocketSendBufSize(1024);
+ Thread threadId[] = new Thread[numThreads];
+ for (int i = 0; i < numThreads; i++) {
+ Transactions trans = new Transactions(proxy, datasize);
+ threadId[i] = new Thread(trans, "TransactionThread-" + i);
+ threadId[i].start();
}
- }
+ // wait for all transactions to get over
+ System.out.println("Waiting for all threads to finish RPCs...");
+ for (int i = 0; i < numThreads; i++) {
+ try {
+ threadId[i].join();
+ } catch (InterruptedException e) {
+ i--; // retry
+ }
+ }
} finally {
- server.stop();
- if(proxy!=null) RPC.stopProxy(proxy);
+ stop(server, proxy);
}
}
-
+
@Test
- public void testStandaloneClient() throws IOException {
+ public void testClientWithoutServer() throws Exception {
+ TestRpcService proxy;
+
+ short invalidPort = 20;
+ InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
+ invalidPort);
+ long invalidClientVersion = 1L;
try {
- TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
- TestProtocol.versionID, new InetSocketAddress(ADDRESS, 20), conf, 15000L);
- proxy.echo("");
+ proxy = RPC.getProxy(TestRpcService.class,
+ invalidClientVersion, invalidAddress, conf);
+ // Test echo method
+ proxy.echo(null, newEchoRequest("hello"));
fail("We should not have reached here");
- } catch (ConnectException ioe) {
+ } catch (ServiceException ioe) {
//this is what we expected
+ if (!(ioe.getCause() instanceof ConnectException)) {
+ fail("We should not have reached here");
+ }
}
}
-
+
private static final String ACL_CONFIG = "test.protocol.acl";
-
+
private static class TestPolicyProvider extends PolicyProvider {
@Override
public Service[] getServices() {
- return new Service[] { new Service(ACL_CONFIG, TestProtocol.class) };
+ return new Service[] { new Service(ACL_CONFIG, TestRpcService.class) };
}
-
}
-
- private void doRPCs(Configuration conf, boolean expectFailure) throws IOException {
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
- server.refreshServiceAcl(conf, new TestPolicyProvider());
+ private void doRPCs(Configuration myConf, boolean expectFailure) throws Exception {
+ Server server;
+ TestRpcService proxy = null;
- TestProtocol proxy = null;
+ server = setupTestServer(myConf, 5);
- server.start();
+ server.refreshServiceAcl(myConf, new TestPolicyProvider());
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
- try {
- proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
- proxy.ping();
+ TestProtos.EmptyRequestProto emptyRequestProto =
+ TestProtos.EmptyRequestProto.newBuilder().build();
+ try {
+ proxy = getClient(addr, conf);
+ proxy.ping(null, emptyRequestProto);
if (expectFailure) {
fail("Expect RPC.getProxy to fail with AuthorizationException!");
}
- } catch (RemoteException e) {
+ } catch (ServiceException e) {
if (expectFailure) {
- assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
- assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
+ RemoteException re = (RemoteException) e.getCause();
+ assertTrue(re.unwrapRemoteException() instanceof AuthorizationException);
+ assertEquals("RPC error code should be UNAUTHORIZED",
+ RpcErrorCodeProto.FATAL_UNAUTHORIZED, re.getErrorCode());
} else {
throw e;
}
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
if (expectFailure) {
assertCounter("RpcAuthorizationFailures", 1L, rb);
@@ -618,57 +572,58 @@ public class TestRPC {
// 0 for the authentication successes and 0 for failure
assertCounter("RpcAuthenticationFailures", 0L, rb);
assertCounter("RpcAuthenticationSuccesses", 0L, rb);
+
+ stop(server, proxy);
}
}
-
+
@Test
public void testServerAddress() throws IOException {
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
- InetSocketAddress bindAddr = null;
+ Server server;
+
+ server = setupTestServer(conf, 5);
try {
- bindAddr = NetUtils.getConnectAddress(server);
+ InetSocketAddress bindAddr = NetUtils.getConnectAddress(server);
+ assertEquals(InetAddress.getLocalHost(), bindAddr.getAddress());
} finally {
- server.stop();
+ stop(server, null);
}
- assertEquals(InetAddress.getLocalHost(), bindAddr.getAddress());
}
@Test
- public void testAuthorization() throws IOException {
- Configuration conf = new Configuration();
- conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+ public void testAuthorization() throws Exception {
+ Configuration myConf = new Configuration();
+ myConf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
true);
// Expect to succeed
- conf.set(ACL_CONFIG, "*");
- doRPCs(conf, false);
+ myConf.set(ACL_CONFIG, "*");
+ doRPCs(myConf, false);
// Reset authorization to expect failure
- conf.set(ACL_CONFIG, "invalid invalid");
- doRPCs(conf, true);
+ myConf.set(ACL_CONFIG, "invalid invalid");
+ doRPCs(myConf, true);
- conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, 2);
+ myConf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, 2);
// Expect to succeed
- conf.set(ACL_CONFIG, "*");
- doRPCs(conf, false);
+ myConf.set(ACL_CONFIG, "*");
+ doRPCs(myConf, false);
// Reset authorization to expect failure
- conf.set(ACL_CONFIG, "invalid invalid");
- doRPCs(conf, true);
+ myConf.set(ACL_CONFIG, "invalid invalid");
+ doRPCs(myConf, true);
}
/**
* Switch off setting socketTimeout values on RPC sockets.
* Verify that RPC calls still work ok.
*/
- public void testNoPings() throws IOException {
+ public void testNoPings() throws Exception {
Configuration conf = new Configuration();
-
+
conf.setBoolean("ipc.client.ping", false);
new TestRPC().testCallsInternal(conf);
-
+
conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, 2);
new TestRPC().testCallsInternal(conf);
}
@@ -693,6 +648,9 @@ public class TestRPC {
@Test
public void testStopProxy() throws IOException {
+ RPC.setProtocolEngine(conf,
+ StoppedProtocol.class, StoppedRpcEngine.class);
+
StoppedProtocol proxy = RPC.getProxy(StoppedProtocol.class,
StoppedProtocol.versionID, null, conf);
StoppedInvocationHandler invocationHandler = (StoppedInvocationHandler)
@@ -709,8 +667,8 @@ public class TestRPC {
StoppedInvocationHandler invocationHandler = (StoppedInvocationHandler)
Proxy.getInvocationHandler(wrappedProxy);
- StoppedProtocol proxy = (StoppedProtocol) RetryProxy.create(StoppedProtocol.class,
- wrappedProxy, RetryPolicies.RETRY_FOREVER);
+ StoppedProtocol proxy = (StoppedProtocol) RetryProxy.create(
+ StoppedProtocol.class, wrappedProxy, RetryPolicies.RETRY_FOREVER);
assertEquals(0, invocationHandler.getCloseCalled());
RPC.stopProxy(proxy);
@@ -719,101 +677,71 @@ public class TestRPC {
@Test
public void testErrorMsgForInsecureClient() throws IOException {
+ Server server;
+ TestRpcService proxy = null;
+
Configuration serverConf = new Configuration(conf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS,
- serverConf);
+ serverConf);
UserGroupInformation.setConfiguration(serverConf);
- final Server server = new RPC.Builder(serverConf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
- server.start();
+ server = setupTestServer(serverConf, 5);
- UserGroupInformation.setConfiguration(conf);
boolean succeeded = false;
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- TestProtocol proxy = null;
+
try {
- proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
- proxy.echo("");
- } catch (RemoteException e) {
- LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
- assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
- assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
+ UserGroupInformation.setConfiguration(conf);
+ proxy = getClient(addr, conf);
+ proxy.echo(null, newEchoRequest(""));
+ } catch (ServiceException e) {
+ assertTrue(e.getCause() instanceof RemoteException);
+ RemoteException re = (RemoteException) e.getCause();
+ LOG.info("LOGGING MESSAGE: " + re.getLocalizedMessage());
+ assertEquals("RPC error code should be UNAUTHORIZED",
+ RpcErrorCodeProto.FATAL_UNAUTHORIZED, re.getErrorCode());
+ assertTrue(re.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, proxy);
}
assertTrue(succeeded);
conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, 2);
UserGroupInformation.setConfiguration(serverConf);
- final Server multiServer = new RPC.Builder(serverConf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .build();
- multiServer.start();
+ server = setupTestServer(serverConf, 5);
succeeded = false;
- final InetSocketAddress mulitServerAddr =
- NetUtils.getConnectAddress(multiServer);
proxy = null;
try {
UserGroupInformation.setConfiguration(conf);
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, mulitServerAddr, conf);
- proxy.echo("");
- } catch (RemoteException e) {
- LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
- assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
- assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
+ proxy = getClient(addr, conf);
+ proxy.echo(null, newEchoRequest(""));
+ } catch (ServiceException e) {
+ RemoteException re = (RemoteException) e.getCause();
+ LOG.info("LOGGING MESSAGE: " + re.getLocalizedMessage());
+ assertEquals("RPC error code should be UNAUTHORIZED",
+ RpcErrorCodeProto.FATAL_UNAUTHORIZED, re.getErrorCode());
+ assertTrue(re.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
- multiServer.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, proxy);
}
assertTrue(succeeded);
}
/**
- * Count the number of threads that have a stack frame containing
- * the given string
- */
- private static int countThreads(String search) {
- ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
-
- int count = 0;
- ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20);
- for (ThreadInfo info : infos) {
- if (info == null) continue;
- for (StackTraceElement elem : info.getStackTrace()) {
- if (elem.getClassName().contains(search)) {
- count++;
- break;
- }
- }
- }
- return count;
- }
-
- /**
* Test that server.stop() properly stops all threads
*/
@Test
public void testStopsAllThreads() throws IOException, InterruptedException {
+ Server server;
+
int threadsBefore = countThreads("Server$Listener$Reader");
assertEquals("Expect no Reader threads running before test",
- 0, threadsBefore);
+ 0, threadsBefore);
+
+ server = setupTestServer(conf, 5);
- final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
- server.start();
try {
// Wait for at least one reader thread to start
int threadsRunning = 0;
@@ -830,11 +758,12 @@ public class TestRPC {
} finally {
server.stop();
}
+
int threadsAfter = countThreads("Server$Listener$Reader");
assertEquals("Expect no Reader threads left running after test",
- 0, threadsAfter);
+ 0, threadsAfter);
}
-
+
@Test
public void testRPCBuilder() throws IOException {
// Test mandatory field conf
@@ -870,81 +799,79 @@ public class TestRPC {
}
}
}
-
+
@Test(timeout=90000)
- public void testRPCInterruptedSimple() throws IOException {
- final Configuration conf = new Configuration();
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS)
- .setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(null).build();
-
- server.start();
+ public void testRPCInterruptedSimple() throws Exception {
+ Server server;
+ TestRpcService proxy = null;
+
+ RPC.Builder builder = newServerBuilder(conf)
+ .setNumHandlers(5).setVerbose(true)
+ .setSecretManager(null);
+
+ server = setupTestServer(builder);
+
try {
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
- final TestProtocol proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
+ proxy = getClient(addr, conf);
// Connect to the server
- proxy.ping();
+
+ proxy.ping(null, newEmptyRequest());
// Interrupt self, try another call
Thread.currentThread().interrupt();
try {
- proxy.ping();
+ proxy.ping(null, newEmptyRequest());
fail("Interruption did not cause IPC to fail");
- } catch (IOException ioe) {
- if (ioe.toString().contains("InterruptedException") ||
- ioe instanceof InterruptedIOException) {
+ } catch (ServiceException se) {
+ if (se.toString().contains("InterruptedException") ||
+ se.getCause() instanceof InterruptedIOException) {
// clear interrupt status for future tests
Thread.interrupted();
return;
}
- throw ioe;
+ throw se;
}
} finally {
- server.stop();
+ stop(server, proxy);
}
}
-
+
@Test(timeout=30000)
- public void testRPCInterrupted() throws IOException, InterruptedException {
- final Configuration conf = new Configuration();
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS)
- .setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(null).build();
-
- server.start();
+ public void testRPCInterrupted() throws Exception {
+ Server server;
+
+ RPC.Builder builder = newServerBuilder(conf)
+ .setNumHandlers(5).setVerbose(true)
+ .setSecretManager(null);
+ server = setupTestServer(builder);
+
+ int numConcurrentRPC = 200;
+ final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
+ final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
+ final AtomicBoolean leaderRunning = new AtomicBoolean(true);
+ final AtomicReference<Throwable> error = new AtomicReference<>();
+ Thread leaderThread = null;
+
try {
- int numConcurrentRPC = 200;
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
- final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
- final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
- final AtomicBoolean leaderRunning = new AtomicBoolean(true);
- final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
- Thread leaderThread = null;
-
for (int i = 0; i < numConcurrentRPC; i++) {
final int num = i;
- final TestProtocol proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
+ final TestRpcService proxy = getClient(addr, conf);
Thread rpcThread = new Thread(new Runnable() {
@Override
public void run() {
try {
barrier.await();
while (num == 0 || leaderRunning.get()) {
- proxy.slowPing(false);
+ proxy.slowPing(null, newSlowPingRequest(false));
}
-
- proxy.slowPing(false);
+
+ proxy.slowPing(null, newSlowPingRequest(false));
} catch (Exception e) {
if (num == 0) {
leaderRunning.set(false);
} else {
error.set(e);
}
-
+
LOG.error("thread " + num, e);
} finally {
latch.countDown();
@@ -952,9 +879,9 @@ public class TestRPC {
}
});
rpcThread.start();
-
+
if (leaderThread == null) {
- leaderThread = rpcThread;
+ leaderThread = rpcThread;
}
}
// let threads get past the barrier
@@ -963,9 +890,9 @@ public class TestRPC {
while (leaderRunning.get()) {
leaderThread.interrupt();
}
-
+
latch.await();
-
+
// should not cause any other thread to get an error
assertTrue("rpc got exception " + error.get(), error.get() == null);
} finally {
@@ -975,46 +902,43 @@ public class TestRPC {
@Test
public void testConnectionPing() throws Exception {
- Configuration conf = new Configuration();
+ Server server;
+ TestRpcService proxy = null;
+
int pingInterval = 50;
conf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
- final Server server = new RPC.Builder(conf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .build();
- server.start();
-
- final TestProtocol proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, server.getListenerAddress(), conf);
+ server = setupTestServer(conf, 5);
+
try {
- // this call will throw exception if server couldn't decode the ping
- proxy.sleep(pingInterval*4);
+ proxy = getClient(addr, conf);
+
+ proxy.sleep(null, newSleepRequest(pingInterval * 4));
} finally {
- if (proxy != null) RPC.stopProxy(proxy);
- server.stop();
+ stop(server, proxy);
}
}
@Test
public void testRpcMetrics() throws Exception {
- Configuration configuration = new Configuration();
+ Server server;
+ TestRpcService proxy = null;
+
final int interval = 1;
- configuration.setBoolean(CommonConfigurationKeys.
+ conf.setBoolean(CommonConfigurationKeys.
RPC_METRICS_QUANTILE_ENABLE, true);
- configuration.set(CommonConfigurationKeys.
+ conf.set(CommonConfigurationKeys.
RPC_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
- final Server server = new RPC.Builder(configuration)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .build();
- server.start();
- final TestProtocol proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, server.getListenerAddress(), configuration);
+
+ server = setupTestServer(conf, 5);
+
try {
- for (int i=0; i<1000; i++) {
- proxy.ping();
- proxy.echo("" + i);
+ proxy = getClient(addr, conf);
+
+ for (int i = 0; i < 1000; i++) {
+ proxy.ping(null, newEmptyRequest());
+
+ proxy.echo(null, newEchoRequest("" + i));
}
MetricsRecordBuilder rpcMetrics =
getMetrics(server.getRpcMetrics().name());
@@ -1027,71 +951,7 @@ public class TestRPC {
MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s",
rpcMetrics);
} finally {
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
- server.stop();
- }
- }
-
- /**
- * Verify the RPC server can shutdown properly when callQueue is full.
- */
- @Test (timeout=30000)
- public void testRPCServerShutdown() throws Exception {
- final int numClients = 3;
- final List<Future<Void>> res = new ArrayList<Future<Void>>();
- final ExecutorService executorService =
- Executors.newFixedThreadPool(numClients);
- final Configuration conf = new Configuration();
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- final Server server = new RPC.Builder(conf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0)
- .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true)
- .build();
- server.start();
-
- final TestProtocol proxy =
- RPC.getProxy(TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
- try {
- // start a sleep RPC call to consume the only handler thread.
- // Start another sleep RPC call to make callQueue full.
- // Start another sleep RPC call to make reader thread block on CallQueue.
- for (int i = 0; i < numClients; i++) {
- res.add(executorService.submit(
- new Callable<Void>() {
- @Override
- public Void call() throws IOException, InterruptedException {
- proxy.sleep(100000);
- return null;
- }
- }));
- }
- while (server.getCallQueueLen() != 1
- || countThreads(CallQueueManager.class.getName()) != 1
- || countThreads(TestImpl.class.getName()) != 1) {
- Thread.sleep(100);
- }
- } finally {
- try {
- server.stop();
- assertEquals("Not enough clients", numClients, res.size());
- for (Future<Void> f : res) {
- try {
- f.get();
- fail("Future get should not return");
- } catch (ExecutionException e) {
- assertTrue("Unexpected exception: " + e,
- e.getCause() instanceof IOException);
- LOG.info("Expected exception", e.getCause());
- }
- }
- } finally {
- RPC.stopProxy(proxy);
- executorService.shutdown();
- }
+ stop(server, proxy);
}
}
@@ -1100,30 +960,28 @@ public class TestRPC {
*/
@Test (timeout=30000)
public void testClientBackOff() throws Exception {
+ Server server;
+ final TestRpcService proxy;
+
boolean succeeded = false;
final int numClients = 2;
final List<Future<Void>> res = new ArrayList<Future<Void>>();
final ExecutorService executorService =
Executors.newFixedThreadPool(numClients);
- final Configuration conf = new Configuration();
conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
conf.setBoolean(CommonConfigurationKeys.IPC_CALLQUEUE_NAMESPACE +
".0." + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true);
- final Server server = new RPC.Builder(conf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0)
- .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true)
- .build();
+ RPC.Builder builder = newServerBuilder(conf)
+ .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
+ server = setupTestServer(builder);
+
@SuppressWarnings("unchecked")
CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox
.getInternalState(server, "callQueue"));
Whitebox.setInternalState(server, "callQueue", spy);
- server.start();
Exception lastException = null;
- final TestProtocol proxy =
- RPC.getProxy(TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
+ proxy = getClient(addr, conf);
try {
// start a sleep RPC call to consume the only handler thread.
// Start another sleep RPC call to make callQueue full.
@@ -1132,27 +990,27 @@ public class TestRPC {
res.add(executorService.submit(
new Callable<Void>() {
@Override
- public Void call() throws IOException, InterruptedException {
- proxy.sleep(100000);
+ public Void call() throws ServiceException, InterruptedException {
+ proxy.sleep(null, newSleepRequest(100000));
return null;
}
}));
verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
}
try {
- proxy.sleep(100);
- } catch (RemoteException e) {
- IOException unwrapExeption = e.unwrapRemoteException();
+ proxy.sleep(null, newSleepRequest(100));
+ } catch (ServiceException e) {
+ RemoteException re = (RemoteException) e.getCause();
+ IOException unwrapExeption = re.unwrapRemoteException();
if (unwrapExeption instanceof RetriableException) {
- succeeded = true;
+ succeeded = true;
} else {
lastException = unwrapExeption;
}
}
} finally {
- server.stop();
- RPC.stopProxy(proxy);
executorService.shutdown();
+ stop(server, proxy);
}
if (lastException != null) {
LOG.error("Last received non-RetriableException:", lastException);
@@ -1165,32 +1023,27 @@ public class TestRPC {
*/
@Test(timeout=30000)
public void testClientRpcTimeout() throws Exception {
- final Server server = new RPC.Builder(conf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0)
- .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true)
- .build();
- server.start();
-
- final Configuration conf = new Configuration();
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
- final TestProtocol proxy =
- RPC.getProxy(TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
+ Server server;
+ TestRpcService proxy = null;
+ RPC.Builder builder = newServerBuilder(conf)
+ .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
+ server = setupTestServer(builder);
+
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
try {
- proxy.sleep(3000);
+ proxy = getClient(addr, conf);
+ proxy.sleep(null, newSleepRequest(3000));
fail("RPC should time out.");
- } catch (SocketTimeoutException e) {
+ } catch (ServiceException e) {
+ assertTrue(e.getCause() instanceof SocketTimeoutException);
LOG.info("got expected timeout.", e);
} finally {
- server.stop();
- RPC.stopProxy(proxy);
+ stop(server, proxy);
}
}
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
new TestRPC().testCallsInternal(conf);
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCServerShutdown.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCServerShutdown.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCServerShutdown.java
new file mode 100644
index 0000000..93af7d4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCServerShutdown.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Split from TestRPC. */
+@SuppressWarnings("deprecation")
+public class TestRPCServerShutdown extends TestRpcBase {
+
+ public static final Log LOG = LogFactory.getLog(TestRPCServerShutdown.class);
+
+ @Before
+ public void setup() {
+ setupConf();
+ }
+
+ /**
+ * Verify the RPC server can shutdown properly when callQueue is full.
+ */
+ @Test (timeout=30000)
+ public void testRPCServerShutdown() throws Exception {
+ final int numClients = 3;
+ final List<Future<Void>> res = new ArrayList<Future<Void>>();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(numClients);
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ RPC.Builder builder = newServerBuilder(conf)
+ .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
+ final Server server = setupTestServer(builder);
+
+ final TestRpcService proxy = getClient(addr, conf);
+ try {
+ // start a sleep RPC call to consume the only handler thread.
+ // Start another sleep RPC call to make callQueue full.
+ // Start another sleep RPC call to make reader thread block on CallQueue.
+ for (int i = 0; i < numClients; i++) {
+ res.add(executorService.submit(
+ new Callable<Void>() {
+ @Override
+ public Void call() throws ServiceException, InterruptedException {
+ proxy.sleep(null, newSleepRequest(100000));
+ return null;
+ }
+ }));
+ }
+ while (server.getCallQueueLen() != 1
+ || countThreads(CallQueueManager.class.getName()) != 1
+ || countThreads(PBServerImpl.class.getName()) != 1) {
+ Thread.sleep(100);
+ }
+ } finally {
+ try {
+ stop(server, proxy);
+ assertEquals("Not enough clients", numClients, res.size());
+ for (Future<Void> f : res) {
+ try {
+ f.get();
+ fail("Future get should not return");
+ } catch (ExecutionException e) {
+ ServiceException se = (ServiceException) e.getCause();
+ assertTrue("Unexpected exception: " + se,
+ se.getCause() instanceof IOException);
+ LOG.info("Expected exception", e.getCause());
+ }
+ }
+ } finally {
+ executorService.shutdown();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
new file mode 100644
index 0000000..03fd31e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.junit.Assert;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+/** Test facilities for unit tests for RPC. */
+public class TestRpcBase {
+
+ protected final static String SERVER_PRINCIPAL_KEY =
+ "test.ipc.server.principal";
+ protected final static String ADDRESS = "0.0.0.0";
+ protected final static int PORT = 0;
+ protected static InetSocketAddress addr;
+ protected static Configuration conf;
+
+ protected void setupConf() {
+ conf = new Configuration();
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ protected static RPC.Builder newServerBuilder(
+ Configuration serverConf) throws IOException {
+ // Create server side implementation
+ PBServerImpl serverImpl = new PBServerImpl();
+ BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto
+ .newReflectiveBlockingService(serverImpl);
+
+ // Get RPC server for server side implementation
+ RPC.Builder builder = new RPC.Builder(serverConf)
+ .setProtocol(TestRpcService.class)
+ .setInstance(service).setBindAddress(ADDRESS).setPort(PORT);
+
+ return builder;
+ }
+
+ protected static RPC.Server setupTestServer(Configuration serverConf,
+ int numHandlers) throws IOException {
+ return setupTestServer(serverConf, numHandlers, null);
+ }
+
+ protected static RPC.Server setupTestServer(Configuration serverConf,
+ int numHandlers,
+ SecretManager<?> serverSm)
+ throws IOException {
+ RPC.Builder builder = newServerBuilder(serverConf);
+
+ if (numHandlers > 0) {
+ builder.setNumHandlers(numHandlers);
+ }
+
+ if (serverSm != null) {
+ builder.setSecretManager(serverSm);
+ }
+
+ return setupTestServer(builder);
+ }
+
+ protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
+ RPC.Server server = builder.build();
+
+ server.start();
+
+ addr = NetUtils.getConnectAddress(server);
+
+ return server;
+ }
+
+ protected static TestRpcService getClient(InetSocketAddress serverAddr,
+ Configuration clientConf)
+ throws ServiceException {
+ try {
+ return RPC.getProxy(TestRpcService.class, 0, serverAddr, clientConf);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ protected static void stop(Server server, TestRpcService proxy) {
+ if (proxy != null) {
+ try {
+ RPC.stopProxy(proxy);
+ } catch (Exception ignored) {}
+ }
+
+ if (server != null) {
+ try {
+ server.stop();
+ } catch (Exception ignored) {}
+ }
+ }
+
+ /**
+ * Count the number of threads that have a stack frame containing
+ * the given string
+ */
+ protected static int countThreads(String search) {
+ ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
+
+ int count = 0;
+ ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20);
+ for (ThreadInfo info : infos) {
+ if (info == null) continue;
+ for (StackTraceElement elem : info.getStackTrace()) {
+ if (elem.getClassName().contains(search)) {
+ count++;
+ break;
+ }
+ }
+ }
+ return count;
+ }
+
+ @ProtocolInfo(protocolName = "org.apache.hadoop.ipc.TestRpcBase$TestRpcService",
+ protocolVersion = 1)
+ public interface TestRpcService
+ extends TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface {
+ }
+
+ public static class PBServerImpl implements TestRpcService {
+ CountDownLatch fastPingCounter = new CountDownLatch(2);
+ private List<Server.Call> postponedCalls = new ArrayList<>();
+
+ @Override
+ public TestProtos.EmptyResponseProto ping(RpcController unused,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ // Ensure clientId is received
+ byte[] clientId = Server.getClientId();
+ Assert.assertNotNull(clientId);
+ Assert.assertEquals(ClientId.BYTE_LENGTH, clientId.length);
+ return TestProtos.EmptyResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public TestProtos.EchoResponseProto echo(
+ RpcController unused, TestProtos.EchoRequestProto request)
+ throws ServiceException {
+ return TestProtos.EchoResponseProto.newBuilder().setMessage(
+ request.getMessage())
+ .build();
+ }
+
+ @Override
+ public TestProtos.EmptyResponseProto error(
+ RpcController unused, TestProtos.EmptyRequestProto request)
+ throws ServiceException {
+ throw new ServiceException("error", new RpcServerException("error"));
+ }
+
+ @Override
+ public TestProtos.EmptyResponseProto error2(
+ RpcController unused, TestProtos.EmptyRequestProto request)
+ throws ServiceException {
+ throw new ServiceException("error", new URISyntaxException("",
+ "testException"));
+ }
+
+ @Override
+ public TestProtos.EmptyResponseProto slowPing(
+ RpcController unused, TestProtos.SlowPingRequestProto request)
+ throws ServiceException {
+ boolean shouldSlow = request.getShouldSlow();
+ if (shouldSlow) {
+ try {
+ fastPingCounter.await(); //slow response until two fast pings happened
+ } catch (InterruptedException ignored) {}
+ } else {
+ fastPingCounter.countDown();
+ }
+
+ return TestProtos.EmptyResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public TestProtos.EchoResponseProto2 echo2(
+ RpcController controller, TestProtos.EchoRequestProto2 request)
+ throws ServiceException {
+ return TestProtos.EchoResponseProto2.newBuilder().addAllMessage(
+ request.getMessageList()).build();
+ }
+
+ @Override
+ public TestProtos.AddResponseProto add(
+ RpcController controller, TestProtos.AddRequestProto request)
+ throws ServiceException {
+ return TestProtos.AddResponseProto.newBuilder().setResult(
+ request.getParam1() + request.getParam2()).build();
+ }
+
+ @Override
+ public TestProtos.AddResponseProto add2(
+ RpcController controller, TestProtos.AddRequestProto2 request)
+ throws ServiceException {
+ int sum = 0;
+ for (Integer num : request.getParamsList()) {
+ sum += num;
+ }
+ return TestProtos.AddResponseProto.newBuilder().setResult(sum).build();
+ }
+
+ @Override
+ public TestProtos.EmptyResponseProto testServerGet(
+ RpcController controller, TestProtos.EmptyRequestProto request)
+ throws ServiceException {
+ if (!(Server.get() instanceof RPC.Server)) {
+ throw new ServiceException("Server.get() failed");
+ }
+ return TestProtos.EmptyResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public TestProtos.ExchangeResponseProto exchange(
+ RpcController controller, TestProtos.ExchangeRequestProto request)
+ throws ServiceException {
+ Integer[] values = new Integer[request.getValuesCount()];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = i;
+ }
+ return TestProtos.ExchangeResponseProto.newBuilder()
+ .addAllValues(Arrays.asList(values)).build();
+ }
+
+ @Override
+ public TestProtos.EmptyResponseProto sleep(
+ RpcController controller, TestProtos.SleepRequestProto request)
+ throws ServiceException {
+ try {
+ Thread.sleep(request.getMilliSeconds());
+ } catch (InterruptedException ignore) {}
+ return TestProtos.EmptyResponseProto.newBuilder().build();
+ }
+ }
+
+ protected static TestProtos.EmptyRequestProto newEmptyRequest() {
+ return TestProtos.EmptyRequestProto.newBuilder().build();
+ }
+
+ protected static TestProtos.EchoRequestProto newEchoRequest(String msg) {
+ return TestProtos.EchoRequestProto.newBuilder().setMessage(msg).build();
+ }
+
+ protected static String convert(TestProtos.EchoResponseProto response) {
+ return response.getMessage();
+ }
+
+ protected static TestProtos.SlowPingRequestProto newSlowPingRequest(
+ boolean shouldSlow) throws ServiceException {
+ return TestProtos.SlowPingRequestProto.newBuilder().
+ setShouldSlow(shouldSlow).build();
+ }
+
+ protected static TestProtos.SleepRequestProto newSleepRequest(
+ int milliSeconds) {
+ return TestProtos.SleepRequestProto.newBuilder()
+ .setMilliSeconds(milliSeconds).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 4ab590e..ba0038d 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -48,4 +48,37 @@ message SleepRequestProto{
}
message SleepResponseProto{
+}
+
+message SlowPingRequestProto {
+ required bool shouldSlow = 1;
+}
+
+message EchoRequestProto2 {
+ repeated string message = 1;
+}
+
+message EchoResponseProto2 {
+ repeated string message = 1;
+}
+
+message AddRequestProto {
+ required int32 param1 = 1;
+ required int32 param2 = 2;
+}
+
+message AddRequestProto2 {
+ repeated int32 params = 1;
+}
+
+message AddResponseProto {
+ required int32 result = 1;
+}
+
+message ExchangeRequestProto {
+ repeated int32 values = 1;
+}
+
+message ExchangeResponseProto {
+ repeated int32 values = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fa54d45/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 722af89..abb3883 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -32,6 +32,13 @@ service TestProtobufRpcProto {
rpc echo(EchoRequestProto) returns (EchoResponseProto);
rpc error(EmptyRequestProto) returns (EmptyResponseProto);
rpc error2(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc slowPing(SlowPingRequestProto) returns (EmptyResponseProto);
+ rpc echo2(EchoRequestProto2) returns (EchoResponseProto2);
+ rpc add(AddRequestProto) returns (AddResponseProto);
+ rpc add2(AddRequestProto2) returns (AddResponseProto);
+ rpc testServerGet(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
+ rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
}
service TestProtobufRpc2Proto {
[29/50] [abbrv] hadoop git commit: HDFS-8791. block ID-based DN
storage layout can be very slow for datanode on ext4. Contributed by Chris
Trezzo.
Posted by ar...@apache.org.
HDFS-8791. block ID-based DN storage layout can be very slow for datanode on ext4. Contributed by Chris Trezzo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c8496eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c8496eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c8496eb
Branch: refs/heads/HDFS-1312
Commit: 2c8496ebf3b7b31c2e18fdf8d4cb2a0115f43112
Parents: 2137e8f
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Mar 1 14:50:02 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Mar 1 15:04:59 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/datanode/DataNodeLayoutVersion.java | 5 +++-
.../hdfs/server/datanode/DataStorage.java | 11 ++++---
.../hdfs/server/datanode/DatanodeUtil.java | 4 +--
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 29 +++++++++++++++++--
.../resources/hadoop-56-layout-datanode-dir.tgz | Bin 0 -> 198996 bytes
.../resources/hadoop-to-57-dn-layout-dir.txt | 24 +++++++++++++++
7 files changed, 67 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8e3eef7..39650e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2896,6 +2896,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9395. Make HDFS audit logging consistant (Kuhu Shukla via kihwal)
+ HDFS-8791. block ID-based DN storage layout can be very slow for datanode
+ on ext4 (Chris Trezzo via kihwal)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
index 23e7cfe..609a740 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java
@@ -65,7 +65,10 @@ public class DataNodeLayoutVersion {
FIRST_LAYOUT(-55, -53, "First datanode layout", false),
BLOCKID_BASED_LAYOUT(-56,
"The block ID of a finalized block uniquely determines its position " +
- "in the directory structure");
+ "in the directory structure"),
+ BLOCKID_BASED_LAYOUT_32_by_32(-57,
+ "Identical to the block id based layout (-56) except it uses a smaller"
+ + " directory structure (32x32)");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 6697054..3844c8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1102,10 +1102,13 @@ public class DataStorage extends Storage {
LOG.info("Start linking block files from " + from + " to " + to);
boolean upgradeToIdBasedLayout = false;
// If we are upgrading from a version older than the one where we introduced
- // block ID-based layout AND we're working with the finalized directory,
- // we'll need to upgrade from the old flat layout to the block ID-based one
- if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT.getInfo().
- getLayoutVersion() && to.getName().equals(STORAGE_DIR_FINALIZED)) {
+ // block ID-based layout (32x32) AND we're working with the finalized
+ // directory, we'll need to upgrade from the old layout to the new one. The
+ // upgrade path from pre-blockid based layouts (>-56) and blockid based
+ // 256x256 layouts (-56) is fortunately the same.
+ if (oldLV > DataNodeLayoutVersion.Feature.BLOCKID_BASED_LAYOUT_32_by_32
+ .getInfo().getLayoutVersion()
+ && to.getName().equals(STORAGE_DIR_FINALIZED)) {
upgradeToIdBasedLayout = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 746c3f6..ad054a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -113,8 +113,8 @@ public class DatanodeUtil {
* @return
*/
public static File idToBlockDir(File root, long blockId) {
- int d1 = (int)((blockId >> 16) & 0xff);
- int d2 = (int)((blockId >> 8) & 0xff);
+ int d1 = (int) ((blockId >> 16) & 0x1F);
+ int d2 = (int) ((blockId >> 8) & 0x1F);
String path = DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
DataStorage.BLOCK_SUBDIR_PREFIX + d2;
return new File(root, path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
index 224abea..6b60b2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
@@ -28,10 +28,16 @@ public class TestDatanodeLayoutUpgrade {
private static final String HADOOP_DATANODE_DIR_TXT =
"hadoop-datanode-dir.txt";
private static final String HADOOP24_DATANODE = "hadoop-24-datanode-dir.tgz";
+ private static final String HADOOP_56_DN_LAYOUT_TXT =
+ "hadoop-to-57-dn-layout-dir.txt";
+ private static final String HADOOP_56_DN_LAYOUT =
+ "hadoop-56-layout-datanode-dir.tgz";
+ /**
+ * Upgrade from LDir-based layout to 32x32 block ID-based layout (-57) --
+ * change described in HDFS-6482 and HDFS-8791
+ */
@Test
- // Upgrade from LDir-based layout to block ID-based layout -- change described
- // in HDFS-6482
public void testUpgradeToIdBasedLayout() throws IOException {
TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage();
upgrade.unpackStorage(HADOOP24_DATANODE, HADOOP_DATANODE_DIR_TXT);
@@ -45,4 +51,23 @@ public class TestDatanodeLayoutUpgrade {
upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1)
.manageDataDfsDirs(false).manageNameDfsDirs(false), null);
}
+
+ /**
+ * Test upgrade from block ID-based layout 256x256 (-56) to block ID-based
+ * layout 32x32 (-57)
+ */
+ @Test
+ public void testUpgradeFrom256To32Layout() throws IOException {
+ TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage();
+ upgrade.unpackStorage(HADOOP_56_DN_LAYOUT, HADOOP_56_DN_LAYOUT_TXT);
+ Configuration conf = new Configuration(TestDFSUpgradeFromImage.upgradeConf);
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ new File(System.getProperty("test.build.data"), "dfs" + File.separator
+ + "data").toURI().toString());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ new File(System.getProperty("test.build.data"), "dfs" + File.separator
+ + "name").toURI().toString());
+ upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .manageDataDfsDirs(false).manageNameDfsDirs(false), null);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz
new file mode 100644
index 0000000..b43ac17
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-56-layout-datanode-dir.tgz differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c8496eb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
new file mode 100644
index 0000000..2dd1766
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-to-57-dn-layout-dir.txt
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout
+# upgrade test to 32x32 (layout -57)
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+/blocks/part1 286881285
+/blocks/part12922 1068680946
+/blocks/part972 2479788008
+/blocks/part973 1221039573
+overallCRC 1902127725
[23/50] [abbrv] hadoop git commit: YARN-4748.
ApplicationHistoryManagerOnTimelineStore should not swallow exceptions on
generateApplicationReport. Contributed by Li Lu
Posted by ar...@apache.org.
YARN-4748. ApplicationHistoryManagerOnTimelineStore should not swallow exceptions on generateApplicationReport. Contributed by Li Lu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d93c22ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d93c22ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d93c22ec
Branch: refs/heads/HDFS-1312
Commit: d93c22ec274b1a0f29609217039b80732886fed7
Parents: d8f390d
Author: Jian He <ji...@apache.org>
Authored: Mon Feb 29 18:19:09 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Feb 29 18:19:09 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../ApplicationHistoryManagerOnTimelineStore.java | 9 +++++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93c22ec/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 88396af..e6e7af3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1477,6 +1477,9 @@ Release 2.8.0 - UNRELEASED
YARN-4709. NMWebServices produces incorrect JSON for containers.
(Varun Saxena via vvasudev)
+ YARN-4748. ApplicationHistoryManagerOnTimelineStore should not
+ swallow exceptions on generateApplicationReport. (Li Lu via jianhe)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93c22ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 6e6b9fc..fd11aea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -625,6 +625,15 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
} catch (AuthorizationException | ApplicationAttemptNotFoundException e) {
// AuthorizationException is thrown because the user doesn't have access
+ if (e instanceof AuthorizationException) {
+ LOG.warn("Failed to authorize when generating application report for "
+ + app.appReport.getApplicationId()
+ + ". Use a placeholder for its latest attempt id. ", e);
+ } else { // Attempt not found
+ LOG.info("No application attempt found for "
+ + app.appReport.getApplicationId()
+ + ". Use a placeholder for its latest attempt id. ", e);
+ }
// It's possible that the app is finished before the first attempt is created.
app.appReport.setDiagnostics(null);
app.appReport.setCurrentApplicationAttemptId(null);
[45/50] [abbrv] hadoop git commit: HDFS-9048. DistCp documentation is
out-of-dated (Daisuke Kobayashi via iwasakims)
Posted by ar...@apache.org.
HDFS-9048. DistCp documentation is out-of-dated (Daisuke Kobayashi via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33a412e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33a412e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33a412e8
Branch: refs/heads/HDFS-1312
Commit: 33a412e8a4ab729d588a9576fb7eb90239c6e383
Parents: eb864d3
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Thu Mar 3 18:57:23 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Thu Mar 3 18:57:23 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-distcp/src/site/markdown/DistCp.md.vm | 13 +++++++------
2 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a412e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d922cd7..0ddecaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2916,6 +2916,9 @@ Release 2.7.3 - UNRELEASED
HDFS-8791. block ID-based DN storage layout can be very slow for datanode
on ext4 (Chris Trezzo via kihwal)
+ HDFS-9048. DistCp documentation is out-of-dated
+ (Daisuke Kobayashi via iwasakims)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a412e8/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index 8f64ea2..bac5ecc 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -412,12 +412,13 @@ $H3 Map sizing
$H3 Copying Between Versions of HDFS
- For copying between two different versions of Hadoop, one will usually use
- HftpFileSystem. This is a read-only FileSystem, so DistCp must be run on the
- destination cluster (more specifically, on NodeManagers that can write to the
- destination cluster). Each source is specified as
- `hftp://<dfs.http.address>/<path>` (the default `dfs.http.address` is
- `<namenode>:50070`).
+ For copying between two different major versions of Hadoop (e.g. between 1.X
+ and 2.X), one will usually use WebHdfsFileSystem. Unlike the previous
+ HftpFileSystem, as webhdfs is available for both read and write operations,
+ DistCp can be run on both source and destination cluster.
+ Remote cluster is specified as `webhdfs://<namenode_hostname>:<http_port>`.
+ When copying between same major versions of Hadoop cluster (e.g. between 2.X
+ and 2.X), use hdfs protocol for better performance.
$H3 MapReduce and other side-effects
[26/50] [abbrv] hadoop git commit: HADOOP-12851. S3AFileSystem Uptake
of ProviderUtils.excludeIncompatibleCredentialProviders. Contributed by Larry
McCay.
Posted by ar...@apache.org.
HADOOP-12851. S3AFileSystem Uptake of ProviderUtils.excludeIncompatibleCredentialProviders. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d251e554
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d251e554
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d251e554
Branch: refs/heads/HDFS-1312
Commit: d251e55415f1fab085159b9eb3b43214d100b6a8
Parents: 307ec80
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon Feb 29 20:03:42 2016 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon Feb 29 21:59:52 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 47 +++++++++++---------
.../hadoop/fs/s3a/TestS3AConfiguration.java | 34 ++++++++++++++
3 files changed, 63 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d251e554/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b84131b..65767f6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1772,6 +1772,10 @@ Release 2.8.0 - UNRELEASED
HADOOP-12813. Migrate TestRPC and related codes to rebase on
ProtobufRpcEngine. (Kai Zheng via wheat9)
+ HADOOP-12851. S3AFileSystem Uptake of
+ ProviderUtils.excludeIncompatibleCredentialProviders.
+ (Larry McCay via cnauroth)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d251e554/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b9590ea..7ab6c79 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.util.Progressable;
import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -118,16 +119,16 @@ public class S3AFileSystem extends FileSystem {
bucket = name.getHost();
ClientConfiguration awsConf = new ClientConfiguration();
- awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS,
+ awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS,
DEFAULT_MAXIMUM_CONNECTIONS));
boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
DEFAULT_SECURE_CONNECTIONS);
awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES,
+ awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES,
DEFAULT_MAX_ERROR_RETRIES));
awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT,
DEFAULT_ESTABLISH_TIMEOUT));
- awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT,
+ awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT));
String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
if(!signerOverride.isEmpty()) {
@@ -263,9 +264,9 @@ public class S3AFileSystem extends FileSystem {
}
private void initMultipartUploads(Configuration conf) {
- boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
+ boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
DEFAULT_PURGE_EXISTING_MULTIPART);
- long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE,
+ long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE,
DEFAULT_PURGE_EXISTING_MULTIPART_AGE);
if (purgeExistingMultipart) {
@@ -297,9 +298,11 @@ public class S3AFileSystem extends FileSystem {
accessKey = userInfo;
}
}
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
if (accessKey == null) {
try {
- final char[] key = conf.getPassword(ACCESS_KEY);
+ final char[] key = c.getPassword(ACCESS_KEY);
if (key != null) {
accessKey = (new String(key)).trim();
}
@@ -309,7 +312,7 @@ public class S3AFileSystem extends FileSystem {
}
if (secretKey == null) {
try {
- final char[] pass = conf.getPassword(SECRET_KEY);
+ final char[] pass = c.getPassword(SECRET_KEY);
if (pass != null) {
secretKey = (new String(pass)).trim();
}
@@ -390,7 +393,7 @@ public class S3AFileSystem extends FileSystem {
throw new FileNotFoundException("Can't open " + f + " because it is a directory");
}
- return new FSDataInputStream(new S3AInputStream(bucket, pathToKey(f),
+ return new FSDataInputStream(new S3AInputStream(bucket, pathToKey(f),
fileStatus.getLen(), s3, statistics));
}
@@ -425,7 +428,7 @@ public class S3AFileSystem extends FileSystem {
}
// We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file
return new FSDataOutputStream(new S3AOutputStream(getConf(), transfers, this,
- bucket, key, progress, cannedACL, statistics,
+ bucket, key, progress, cannedACL, statistics,
serverSideEncryptionAlgorithm), null);
}
@@ -436,7 +439,7 @@ public class S3AFileSystem extends FileSystem {
* @param progress for reporting progress if it is not null.
* @throws IOException indicating that append is not supported.
*/
- public FSDataOutputStream append(Path f, int bufferSize,
+ public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
throw new IOException("Not supported");
}
@@ -446,8 +449,8 @@ public class S3AFileSystem extends FileSystem {
* Renames Path src to Path dst. Can take place on local fs
* or remote DFS.
*
- * Warning: S3 does not support renames. This method does a copy which can
- * take S3 some time to execute with large files and directories. Since
+ * Warning: S3 does not support renames. This method does a copy which can
+ * take S3 some time to execute with large files and directories. Since
* there is no Progressable passed in, this can time out jobs.
*
* Note: This implementation differs with other S3 drivers. Specifically:
@@ -560,7 +563,7 @@ public class S3AFileSystem extends FileSystem {
return false;
}
- List<DeleteObjectsRequest.KeyVersion> keysToDelete =
+ List<DeleteObjectsRequest.KeyVersion> keysToDelete =
new ArrayList<>();
if (dstStatus != null && dstStatus.isEmptyDirectory()) {
// delete unnecessary fake directory.
@@ -666,7 +669,7 @@ public class S3AFileSystem extends FileSystem {
}
if (!recursive && !status.isEmptyDirectory()) {
- throw new IOException("Path is a folder: " + f +
+ throw new IOException("Path is a folder: " + f +
" and it is not an empty directory");
}
@@ -697,7 +700,7 @@ public class S3AFileSystem extends FileSystem {
//request.setDelimiter("/");
request.setMaxKeys(maxKeys);
- List<DeleteObjectsRequest.KeyVersion> keys =
+ List<DeleteObjectsRequest.KeyVersion> keys =
new ArrayList<>();
ObjectListing objects = s3.listObjects(request);
statistics.incrementReadOps(1);
@@ -801,7 +804,7 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("Adding: fd: " + keyPath);
}
} else {
- result.add(new S3AFileStatus(summary.getSize(),
+ result.add(new S3AFileStatus(summary.getSize(),
dateToLong(summary.getLastModified()), keyPath,
getDefaultBlockSize(f.makeQualified(uri, workingDir))));
if (LOG.isDebugEnabled()) {
@@ -869,7 +872,7 @@ public class S3AFileSystem extends FileSystem {
* @param f path to create
* @param permission to apply to f
*/
- // TODO: If we have created an empty file at /foo/bar and we then call
+ // TODO: If we have created an empty file at /foo/bar and we then call
// mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/?
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
if (LOG.isDebugEnabled()) {
@@ -892,7 +895,7 @@ public class S3AFileSystem extends FileSystem {
FileStatus fileStatus = getFileStatus(fPart);
if (fileStatus.isFile()) {
throw new FileAlreadyExistsException(String.format(
- "Can't make directory for path '%s' since it is a file.",
+ "Can't make directory for path '%s' since it is a file.",
fPart));
}
} catch (FileNotFoundException fnfe) {
@@ -998,9 +1001,9 @@ public class S3AFileSystem extends FileSystem {
if (!objects.getCommonPrefixes().isEmpty()
|| objects.getObjectSummaries().size() > 0) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Found path as directory (with /): " +
- objects.getCommonPrefixes().size() + "/" +
- objects.getObjectSummaries().size());
+ LOG.debug("Found path as directory (with /): " +
+ objects.getCommonPrefixes().size() + "/" +
+ objects.getObjectSummaries().size());
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize());
@@ -1046,7 +1049,7 @@ public class S3AFileSystem extends FileSystem {
* @param dst path
*/
@Override
- public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
String key = pathToKey(dst);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d251e554/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
index 3db84db..e74ebca 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
@@ -29,6 +29,7 @@ import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -318,4 +319,37 @@ public class TestS3AConfiguration {
assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
}
+
+ @Test
+ public void testExcludingS3ACredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ "jceks://s3a/foobar," + jks.toString());
+
+ // first make sure that the s3a based provider is removed
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ String newPath = conf.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
+ assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
+
+ // now let's make sure the new path is created by the S3AFileSystem
+ // and the integration still works. Let's provision the keys through
+ // the altered configuration instance and then try and access them
+ // using the original config with the s3a provider in the path.
+ provisionAccessKeys(c);
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", "456", creds.getAccessSecret());
+
+ }
}
[28/50] [abbrv] hadoop git commit: HDFS-9870. Remove unused imports
from DFSUtil. Contributed by Brahma Reddy Battula.
Posted by ar...@apache.org.
HDFS-9870. Remove unused imports from DFSUtil. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2137e8fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2137e8fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2137e8fe
Branch: refs/heads/HDFS-1312
Commit: 2137e8feeb5c5c88d3a80db3a334fd472f299ee4
Parents: 44d9bac
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Mar 1 11:15:13 2016 -0800
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Mar 1 11:15:13 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java | 3 ---
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2137e8fe/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bb847a1..8e3eef7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2861,6 +2861,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9549. TestCacheDirectives#testExceedsCapacity is flaky (Xiao Chen via
cmccabe)
+ HDFS-9870. Remove unused imports from DFSUtil.
+ (Brahma Reddy Battula via cnauroth)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2137e8fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index b697a9c..1a0d192 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -27,7 +27,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DE
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
@@ -37,7 +36,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PAS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
@@ -55,7 +53,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
-import com.google.common.collect.Sets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.Option;
[41/50] [abbrv] hadoop git commit: HDFS-9835. OIV: add ReverseXML
processor which reconstructs an fsimage from an XML file (cmccabe)
Posted by ar...@apache.org.
HDFS-9835. OIV: add ReverseXML processor which reconstructs an fsimage from an XML file (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/700b0e40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/700b0e40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/700b0e40
Branch: refs/heads/HDFS-1312
Commit: 700b0e4019cf483f7532609711812150b8c44742
Parents: 67880cc
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Feb 25 16:43:54 2016 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Mar 2 17:56:11 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/test/GenericTestUtils.java | 56 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/namenode/FSImageFormatPBINode.java | 12 +-
.../OfflineImageReconstructor.java | 1639 ++++++++++++++++++
.../OfflineImageViewerPB.java | 22 +-
.../offlineImageViewer/PBImageXmlWriter.java | 248 ++-
.../TestOfflineImageViewer.java | 56 +-
7 files changed, 1983 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 7e494a8..1907094 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -17,8 +17,12 @@
*/
package org.apache.hadoop.test;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.StringWriter;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
@@ -32,6 +36,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
@@ -481,4 +486,55 @@ public abstract class GenericTestUtils {
Assume.assumeTrue(
Boolean.parseBoolean(System.getProperty("runningWithNative", "false")));
}
+
+ /**
+ * Get the diff between two files.
+ *
+ * @param a
+ * @param b
+ * @return The empty string if there is no diff; the diff, otherwise.
+ *
+ * @throws IOException If there is an error reading either file.
+ */
+ public static String getFilesDiff(File a, File b) throws IOException {
+ StringBuilder bld = new StringBuilder();
+ BufferedReader ra = null, rb = null;
+ try {
+ ra = new BufferedReader(
+ new InputStreamReader(new FileInputStream(a)));
+ rb = new BufferedReader(
+ new InputStreamReader(new FileInputStream(b)));
+ while (true) {
+ String la = ra.readLine();
+ String lb = rb.readLine();
+ if (la == null) {
+ if (lb != null) {
+ addPlusses(bld, ra);
+ }
+ break;
+ } else if (lb == null) {
+ if (la != null) {
+ addPlusses(bld, rb);
+ }
+ break;
+ }
+ if (!la.equals(lb)) {
+ bld.append(" - ").append(la).append("\n");
+ bld.append(" + ").append(lb).append("\n");
+ }
+ }
+ } finally {
+ IOUtils.closeQuietly(ra);
+ IOUtils.closeQuietly(rb);
+ }
+ return bld.toString();
+ }
+
+ private static void addPlusses(StringBuilder bld, BufferedReader r)
+ throws IOException {
+ String l;
+ while ((l = r.readLine()) != null) {
+ bld.append(" + ").append(l).append("\n");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ee204c8..728342a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -980,6 +980,8 @@ Release 2.9.0 - UNRELEASED
HDFS-9047. Retire libwebhdfs. (wheat9)
NEW FEATURES
+ HDFS-9835. OIV: add ReverseXML processor which reconstructs an fsimage from
+ an XML file (cmccabe)
IMPROVEMENTS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 2f74a2b..1c37b62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -90,14 +90,14 @@ public final class FSImageFormatPBINode {
private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
.values();
- private static final int XATTR_NAMESPACE_MASK = 3;
- private static final int XATTR_NAMESPACE_OFFSET = 30;
- private static final int XATTR_NAME_MASK = (1 << 24) - 1;
- private static final int XATTR_NAME_OFFSET = 6;
+ public static final int XATTR_NAMESPACE_MASK = 3;
+ public static final int XATTR_NAMESPACE_OFFSET = 30;
+ public static final int XATTR_NAME_MASK = (1 << 24) - 1;
+ public static final int XATTR_NAME_OFFSET = 6;
/* See the comments in fsimage.proto for an explanation of the following. */
- private static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
- private static final int XATTR_NAMESPACE_EXT_MASK = 1;
+ public static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
+ public static final int XATTR_NAMESPACE_EXT_MASK = 1;
private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
XAttr.NameSpace.values();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
new file mode 100644
index 0000000..e5d0e2c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
@@ -0,0 +1,1639 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_OFFSET;
+import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_MASK;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+
+import com.google.common.io.CountingOutputStream;
+import com.google.common.primitives.Ints;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.TextFormat;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
+import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection.DiffEntry;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.hdfs.util.XMLUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OfflineImageReconstructor {
+ public static final Log LOG =
+ LogFactory.getLog(OfflineImageReconstructor.class);
+
+ /**
+ * The output stream.
+ */
+ private final CountingOutputStream out;
+
+ /**
+ * A source of XML events based on the input file.
+ */
+ private final XMLEventReader events;
+
+ /**
+ * A map of section names to section handler objects.
+ */
+ private final HashMap<String, SectionProcessor> sections;
+
+ /**
+ * The offset of the start of the current section.
+ */
+ private long sectionStartOffset;
+
+ /**
+ * The FileSummary builder, where we gather information about each section
+ * we wrote.
+ */
+ private final FileSummary.Builder fileSummaryBld =
+ FileSummary.newBuilder();
+
+ /**
+ * The string table. See registerStringId for details.
+ */
+ private final HashMap<String, Integer> stringTable = new HashMap<>();
+
+ /**
+ * The date formatter to use with fsimage XML files.
+ */
+ private final SimpleDateFormat isoDateFormat;
+
+ /**
+ * The latest string ID. See registerStringId for details.
+ */
+ private int latestStringId = 0;
+
+ private OfflineImageReconstructor(CountingOutputStream out,
+ InputStreamReader reader) throws XMLStreamException {
+ this.out = out;
+ XMLInputFactory factory = XMLInputFactory.newInstance();
+ this.events = factory.createXMLEventReader(reader);
+ this.sections = new HashMap<>();
+ this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor());
+ this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor());
+ this.sections.put(SecretManagerSectionProcessor.NAME,
+ new SecretManagerSectionProcessor());
+ this.sections.put(CacheManagerSectionProcessor.NAME,
+ new CacheManagerSectionProcessor());
+ this.sections.put(SnapshotDiffSectionProcessor.NAME,
+ new SnapshotDiffSectionProcessor());
+ this.sections.put(INodeReferenceSectionProcessor.NAME,
+ new INodeReferenceSectionProcessor());
+ this.sections.put(INodeDirectorySectionProcessor.NAME,
+ new INodeDirectorySectionProcessor());
+ this.sections.put(FilesUnderConstructionSectionProcessor.NAME,
+ new FilesUnderConstructionSectionProcessor());
+ this.sections.put(SnapshotSectionProcessor.NAME,
+ new SnapshotSectionProcessor());
+ this.isoDateFormat = PBImageXmlWriter.createSimpleDateFormat();
+ }
+
+ /**
+ * Read the next tag start or end event.
+ *
+ * @param expected The name of the next tag we expect.
+ * We will validate that the tag has this name,
+ * unless this string is enclosed in braces.
+ * @param allowEnd If true, we will also end events.
+ * If false, end events cause an exception.
+ *
+ * @return The next tag start or end event.
+ */
+ private XMLEvent expectTag(String expected, boolean allowEnd)
+ throws IOException {
+ XMLEvent ev = null;
+ while (true) {
+ try {
+ ev = events.nextEvent();
+ } catch (XMLStreamException e) {
+ throw new IOException("Expecting " + expected +
+ ", but got XMLStreamException", e);
+ }
+ switch (ev.getEventType()) {
+ case XMLEvent.ATTRIBUTE:
+ throw new IOException("Got unexpected attribute: " + ev);
+ case XMLEvent.CHARACTERS:
+ if (!ev.asCharacters().isWhiteSpace()) {
+ throw new IOException("Got unxpected characters while " +
+ "looking for " + expected + ": " +
+ ev.asCharacters().getData());
+ }
+ break;
+ case XMLEvent.END_ELEMENT:
+ if (!allowEnd) {
+ throw new IOException("Got unexpected end event " +
+ "while looking for " + expected);
+ }
+ return ev;
+ case XMLEvent.START_ELEMENT:
+ if (!expected.startsWith("[")) {
+ if (!ev.asStartElement().getName().getLocalPart().
+ equals(expected)) {
+ throw new IOException("Failed to find <" + expected + ">; " +
+ "got " + ev.asStartElement().getName().getLocalPart() +
+ " instead.");
+ }
+ }
+ return ev;
+ default:
+ // Ignore other event types like comment, etc.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Skipping XMLEvent of type " +
+ ev.getEventType() + "(" + ev + ")");
+ }
+ break;
+ }
+ }
+ }
+
+ private void expectTagEnd(String expected) throws IOException {
+ XMLEvent ev = expectTag(expected, true);
+ if (ev.getEventType() != XMLStreamConstants.END_ELEMENT) {
+ throw new IOException("Expected tag end event for " + expected +
+ ", but got: " + ev);
+ }
+ if (!expected.startsWith("[")) {
+ String tag = ev.asEndElement().getName().getLocalPart();
+ if (!tag.equals(expected)) {
+ throw new IOException("Expected tag end event for " + expected +
+ ", but got tag end event for " + tag);
+ }
+ }
+ }
+
+ private static class Node {
+ private static final String EMPTY = "";
+ HashMap<String, LinkedList<Node>> children;
+ String val = EMPTY;
+
+ void addChild(String key, Node node) {
+ if (children == null) {
+ children = new HashMap<>();
+ }
+ LinkedList<Node> cur = children.get(key);
+ if (cur == null) {
+ cur = new LinkedList<>();
+ children.put(key, cur);
+ }
+ cur.add(node);
+ }
+
+ Node removeChild(String key) {
+ if (children == null) {
+ return null;
+ }
+ LinkedList<Node> cur = children.get(key);
+ if (cur == null) {
+ return null;
+ }
+ Node node = cur.remove();
+ if ((node == null) || cur.isEmpty()) {
+ children.remove(key);
+ }
+ return node;
+ }
+
+ String removeChildStr(String key) {
+ Node child = removeChild(key);
+ if (child == null) {
+ return null;
+ }
+ if ((child.children != null) && (!child.children.isEmpty())) {
+ throw new RuntimeException("Node " + key + " contains children " +
+ "of its own.");
+ }
+ return child.getVal();
+ }
+
+ Integer removeChildInt(String key) throws IOException {
+ String str = removeChildStr(key);
+ if (str == null) {
+ return null;
+ }
+ return Integer.valueOf(str);
+ }
+
+ Long removeChildLong(String key) throws IOException {
+ String str = removeChildStr(key);
+ if (str == null) {
+ return null;
+ }
+ return Long.valueOf(str);
+ }
+
+ boolean removeChildBool(String key) throws IOException {
+ String str = removeChildStr(key);
+ if (str == null) {
+ return false;
+ }
+ return true;
+ }
+
+ String getRemainingKeyNames() {
+ if (children == null) {
+ return "";
+ }
+ return StringUtils.join(", ", children.keySet());
+ }
+
+ void verifyNoRemainingKeys(String sectionName) throws IOException {
+ String remainingKeyNames = getRemainingKeyNames();
+ if (!remainingKeyNames.isEmpty()) {
+ throw new IOException("Found unknown XML keys in " +
+ sectionName + ": " + remainingKeyNames);
+ }
+ }
+
+ void setVal(String val) {
+ this.val = val;
+ }
+
+ String getVal() {
+ return val;
+ }
+
+ String dump() {
+ StringBuilder bld = new StringBuilder();
+ if ((children != null) && (!children.isEmpty())) {
+ bld.append("{");
+ }
+ if (val != null) {
+ bld.append("[").append(val).append("]");
+ }
+ if ((children != null) && (!children.isEmpty())) {
+ String prefix = "";
+ for (Map.Entry<String, LinkedList<Node>> entry : children.entrySet()) {
+ for (Node n : entry.getValue()) {
+ bld.append(prefix);
+ bld.append(entry.getKey()).append(": ");
+ bld.append(n.dump());
+ prefix = ", ";
+ }
+ }
+ bld.append("}");
+ }
+ return bld.toString();
+ }
+ }
+
+ private void loadNodeChildrenHelper(Node parent, String expected,
+ String terminators[]) throws IOException {
+ XMLEvent ev = null;
+ while (true) {
+ try {
+ ev = events.peek();
+ switch (ev.getEventType()) {
+ case XMLEvent.END_ELEMENT:
+ if (terminators.length != 0) {
+ return;
+ }
+ events.nextEvent();
+ return;
+ case XMLEvent.START_ELEMENT:
+ String key = ev.asStartElement().getName().getLocalPart();
+ for (String terminator : terminators) {
+ if (terminator.equals(key)) {
+ return;
+ }
+ }
+ events.nextEvent();
+ Node node = new Node();
+ parent.addChild(key, node);
+ loadNodeChildrenHelper(node, expected, new String[0]);
+ break;
+ case XMLEvent.CHARACTERS:
+ String val = XMLUtils.
+ unmangleXmlString(ev.asCharacters().getData(), true);
+ parent.setVal(val);
+ events.nextEvent();
+ break;
+ case XMLEvent.ATTRIBUTE:
+ throw new IOException("Unexpected XML event " + ev);
+ default:
+ // Ignore other event types like comment, etc.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Skipping XMLEvent " + ev);
+ }
+ events.nextEvent();
+ break;
+ }
+ } catch (XMLStreamException e) {
+ throw new IOException("Expecting " + expected +
+ ", but got XMLStreamException", e);
+ }
+ }
+ }
+
+ /**
+ * Load a subtree of the XML into a Node structure.
+ * We will keep consuming XML events until we exit the current subtree.
+ * If there are any terminators specified, we will always leave the
+ * terminating end tag event in the stream.
+ *
+ * @param parent The node to fill in.
+ * @param expected A string to display in exceptions.
+ * @param terminators Entering any one of these XML tags terminates our
+ * traversal.
+ * @throws IOException
+ */
+ private void loadNodeChildren(Node parent, String expected,
+ String... terminators) throws IOException {
+ loadNodeChildrenHelper(parent, expected, terminators);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("loadNodeChildren(expected=" + expected +
+ ", terminators=[" + StringUtils.join(",", terminators) +
+ "]):" + parent.dump());
+ }
+ }
+
+ /**
+ * A processor for an FSImage XML section.
+ */
+ private interface SectionProcessor {
+ /**
+ * Process this section.
+ */
+ void process() throws IOException;
+ }
+
+ /**
+ * Processes the NameSection containing last allocated block ID, etc.
+ */
+ private class NameSectionProcessor implements SectionProcessor {
+ static final String NAME = "NameSection";
+
+ @Override
+ public void process() throws IOException {
+ Node node = new Node();
+ loadNodeChildren(node, "NameSection fields");
+ NameSystemSection.Builder b = NameSystemSection.newBuilder();
+ Integer namespaceId = node.removeChildInt("namespaceId");
+ if (namespaceId == null) {
+ throw new IOException("<NameSection> is missing <namespaceId>");
+ }
+ b.setNamespaceId(namespaceId);
+ Long lval = node.removeChildLong("genstampV1");
+ if (lval != null) {
+ b.setGenstampV1(lval);
+ }
+ lval = node.removeChildLong("genstampV2");
+ if (lval != null) {
+ b.setGenstampV2(lval);
+ }
+ lval = node.removeChildLong("genstampV1Limit");
+ if (lval != null) {
+ b.setGenstampV1Limit(lval);
+ }
+ lval = node.removeChildLong("lastAllocatedBlockId");
+ if (lval != null) {
+ b.setLastAllocatedBlockId(lval);
+ }
+ lval = node.removeChildLong("txid");
+ if (lval != null) {
+ b.setTransactionId(lval);
+ }
+ lval = node.removeChildLong("rollingUpgradeStartTime");
+ if (lval != null) {
+ b.setRollingUpgradeStartTime(lval);
+ }
+ lval = node.removeChildLong("lastAllocatedStripedBlockId");
+ if (lval != null) {
+ b.setLastAllocatedStripedBlockId(lval);
+ }
+ node.verifyNoRemainingKeys("NameSection");
+ NameSystemSection s = b.build();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(SectionName.NS_INFO.name() + " writing header: {" +
+ TextFormat.printToString(s) + "}");
+ }
+ s.writeDelimitedTo(out);
+ recordSectionLength(SectionName.NS_INFO.name());
+ }
+ }
+
+ private class INodeSectionProcessor implements SectionProcessor {
+ static final String NAME = "INodeSection";
+
+ @Override
+ public void process() throws IOException {
+ Node headerNode = new Node();
+ loadNodeChildren(headerNode, "INodeSection fields", "inode");
+ INodeSection.Builder b = INodeSection.newBuilder();
+ Long lval = headerNode.removeChildLong("lastInodeId");
+ if (lval != null) {
+ b.setLastInodeId(lval);
+ }
+ Integer expectedNumINodes = headerNode.removeChildInt("numInodes");
+ if (expectedNumINodes == null) {
+ throw new IOException("Failed to find <numInodes> in INodeSection.");
+ }
+ b.setNumInodes(expectedNumINodes);
+ INodeSection s = b.build();
+ s.writeDelimitedTo(out);
+ headerNode.verifyNoRemainingKeys("INodeSection");
+ int actualNumINodes = 0;
+ while (actualNumINodes < expectedNumINodes) {
+ try {
+ expectTag("inode", false);
+ } catch (IOException e) {
+ throw new IOException("Only found " + actualNumINodes +
+ " <inode> entries out of " + expectedNumINodes, e);
+ }
+ actualNumINodes++;
+ Node inode = new Node();
+ loadNodeChildren(inode, "INode fields");
+ INodeSection.INode.Builder inodeBld = processINodeXml(inode);
+ inodeBld.build().writeDelimitedTo(out);
+ }
+ expectTagEnd("INodeSection");
+ recordSectionLength(SectionName.INODE.name());
+ }
+ }
+
+ private INodeSection.INode.Builder processINodeXml(Node node)
+ throws IOException {
+ String type = node.removeChildStr("type");
+ if (type == null) {
+ throw new IOException("INode XML found with no <type> tag.");
+ }
+ INodeSection.INode.Builder inodeBld = INodeSection.INode.newBuilder();
+ Long id = node.removeChildLong("id");
+ if (id == null) {
+ throw new IOException("<inode> found without <id>");
+ }
+ inodeBld.setId(id);
+ String name = node.removeChildStr("name");
+ if (name != null) {
+ inodeBld.setName(ByteString.copyFrom(name, "UTF8"));
+ }
+ switch (type) {
+ case "FILE":
+ processFileXml(node, inodeBld );
+ break;
+ case "DIRECTORY":
+ processDirectoryXml(node, inodeBld);
+ break;
+ case "SYMLINK":
+ processSymlinkXml(node, inodeBld);
+ break;
+ default:
+ throw new IOException("INode XML found with unknown <type> " +
+ "tag " + type);
+ }
+ node.verifyNoRemainingKeys("inode");
+ return inodeBld;
+ }
+
+ private void processFileXml(Node node, INodeSection.INode.Builder inodeBld)
+ throws IOException {
+ inodeBld.setType(INodeSection.INode.Type.FILE);
+ INodeSection.INodeFile.Builder bld = INodeSection.INodeFile.newBuilder();
+ Integer ival = node.removeChildInt("replication");
+ if (ival != null) {
+ bld.setReplication(ival);
+ }
+ Long lval = node.removeChildLong("mtime");
+ if (lval != null) {
+ bld.setModificationTime(lval);
+ }
+ lval = node.removeChildLong("atime");
+ if (lval != null) {
+ bld.setAccessTime(lval);
+ }
+ lval = node.removeChildLong("preferredBlockSize");
+ if (lval != null) {
+ bld.setPreferredBlockSize(lval);
+ }
+ String perm = node.removeChildStr("permission");
+ if (perm != null) {
+ bld.setPermission(permissionXmlToU64(perm));
+ }
+ Node blocks = node.removeChild("blocks");
+ if (blocks != null) {
+ while (true) {
+ Node block = blocks.removeChild("block");
+ if (block == null) {
+ break;
+ }
+ HdfsProtos.BlockProto.Builder blockBld =
+ HdfsProtos.BlockProto.newBuilder();
+ Long id = block.removeChildLong("id");
+ if (id == null) {
+ throw new IOException("<block> found without <id>");
+ }
+ blockBld.setBlockId(id);
+ Long genstamp = block.removeChildLong("genstamp");
+ if (genstamp == null) {
+ throw new IOException("<block> found without <genstamp>");
+ }
+ blockBld.setGenStamp(genstamp);
+ Long numBytes = block.removeChildLong("numBytes");
+ if (numBytes == null) {
+ throw new IOException("<block> found without <numBytes>");
+ }
+ blockBld.setNumBytes(numBytes);
+ bld.addBlocks(blockBld);
+ }
+ }
+ Node fileUnderConstruction = node.removeChild("file-under-construction");
+ if (fileUnderConstruction != null) {
+ INodeSection.FileUnderConstructionFeature.Builder fb =
+ INodeSection.FileUnderConstructionFeature.newBuilder();
+ String clientName =
+ fileUnderConstruction.removeChildStr("clientName");
+ if (clientName == null) {
+ throw new IOException("<file-under-construction> found without " +
+ "<clientName>");
+ }
+ fb.setClientName(clientName);
+ String clientMachine =
+ fileUnderConstruction.removeChildStr("clientMachine");
+ if (clientMachine == null) {
+ throw new IOException("<file-under-construction> found without " +
+ "<clientMachine>");
+ }
+ fb.setClientMachine(clientMachine);
+ bld.setFileUC(fb);
+ }
+ Node acls = node.removeChild("acls");
+ if (acls != null) {
+ bld.setAcl(aclXmlToProto(acls));
+ }
+ Node xattrs = node.removeChild("xattrs");
+ if (xattrs != null) {
+ bld.setXAttrs(xattrsXmlToProto(xattrs));
+ }
+ ival = node.removeChildInt("storagePolicyId");
+ if (ival != null) {
+ bld.setStoragePolicyID(ival);
+ }
+ Boolean bval = node.removeChildBool("isStriped");
+ bld.setIsStriped(bval);
+ inodeBld.setFile(bld);
+ // Will check remaining keys and serialize in processINodeXml
+ }
+
+ private void processDirectoryXml(Node node,
+ INodeSection.INode.Builder inodeBld) throws IOException {
+ inodeBld.setType(INodeSection.INode.Type.DIRECTORY);
+ INodeSection.INodeDirectory.Builder bld =
+ INodeSection.INodeDirectory.newBuilder();
+ Long lval = node.removeChildLong("mtime");
+ if (lval != null) {
+ bld.setModificationTime(lval);
+ }
+ lval = node.removeChildLong("nsquota");
+ if (lval != null) {
+ bld.setNsQuota(lval);
+ }
+ lval = node.removeChildLong("dsquota");
+ if (lval != null) {
+ bld.setDsQuota(lval);
+ }
+ String perm = node.removeChildStr("permission");
+ if (perm != null) {
+ bld.setPermission(permissionXmlToU64(perm));
+ }
+ Node acls = node.removeChild("acls");
+ if (acls != null) {
+ bld.setAcl(aclXmlToProto(acls));
+ }
+ Node xattrs = node.removeChild("xattrs");
+ if (xattrs != null) {
+ bld.setXAttrs(xattrsXmlToProto(xattrs));
+ }
+ INodeSection.QuotaByStorageTypeFeatureProto.Builder qf =
+ INodeSection.QuotaByStorageTypeFeatureProto.newBuilder();
+ while (true) {
+ Node typeQuota = node.removeChild("typeQuota");
+ if (typeQuota == null) {
+ break;
+ }
+ INodeSection.QuotaByStorageTypeEntryProto.Builder qbld =
+ INodeSection.QuotaByStorageTypeEntryProto.newBuilder();
+ String type = typeQuota.removeChildStr("type");
+ if (type == null) {
+ throw new IOException("<typeQuota> was missing <type>");
+ }
+ HdfsProtos.StorageTypeProto storageType =
+ HdfsProtos.StorageTypeProto.valueOf(type);
+ if (storageType == null) {
+ throw new IOException("<typeQuota> had unknown <type> " + type);
+ }
+ qbld.setStorageType(storageType);
+ Long quota = typeQuota.removeChildLong("quota");
+ if (quota == null) {
+ throw new IOException("<typeQuota> was missing <quota>");
+ }
+ qbld.setQuota(quota);
+ qf.addQuotas(qbld);
+ }
+ bld.setTypeQuotas(qf);
+ inodeBld.setDirectory(bld);
+ // Will check remaining keys and serialize in processINodeXml
+ }
+
+ private void processSymlinkXml(Node node,
+ INodeSection.INode.Builder inodeBld) throws IOException {
+ inodeBld.setType(INodeSection.INode.Type.SYMLINK);
+ INodeSection.INodeSymlink.Builder bld =
+ INodeSection.INodeSymlink.newBuilder();
+ String perm = node.removeChildStr("permission");
+ if (perm != null) {
+ bld.setPermission(permissionXmlToU64(perm));
+ }
+ String target = node.removeChildStr("target");
+ if (target != null) {
+ bld.setTarget(ByteString.copyFrom(target, "UTF8"));
+ }
+ Long lval = node.removeChildLong("mtime");
+ if (lval != null) {
+ bld.setModificationTime(lval);
+ }
+ lval = node.removeChildLong("atime");
+ if (lval != null) {
+ bld.setAccessTime(lval);
+ }
+ inodeBld.setSymlink(bld);
+ // Will check remaining keys and serialize in processINodeXml
+ }
+
+ private INodeSection.AclFeatureProto.Builder aclXmlToProto(Node acl)
+ throws IOException {
+ // TODO: support ACLs
+ throw new IOException("ACLs are not supported yet.");
+ }
+
+ private INodeSection.XAttrFeatureProto.Builder xattrsXmlToProto(Node xattrs)
+ throws IOException {
+ INodeSection.XAttrFeatureProto.Builder bld =
+ INodeSection.XAttrFeatureProto.newBuilder();
+ while (true) {
+ Node xattr = xattrs.removeChild("xattr");
+ if (xattr == null) {
+ break;
+ }
+ INodeSection.XAttrCompactProto.Builder b =
+ INodeSection.XAttrCompactProto.newBuilder();
+ String ns = xattr.removeChildStr("ns");
+ if (ns == null) {
+ throw new IOException("<xattr> had no <ns> entry.");
+ }
+ int nsIdx = XAttrProtos.XAttrProto.
+ XAttrNamespaceProto.valueOf(ns).ordinal();
+ String name = xattr.removeChildStr("name");
+ String valStr = xattr.removeChildStr("val");
+ byte[] val = null;
+ if (valStr == null) {
+ String valHex = xattr.removeChildStr("valHex");
+ if (valHex == null) {
+ throw new IOException("<xattr> had no <val> or <valHex> entry.");
+ }
+ val = new HexBinaryAdapter().unmarshal(valHex);
+ } else {
+ val = valStr.getBytes("UTF8");
+ }
+ b.setValue(ByteString.copyFrom(val));
+
+ // The XAttrCompactProto name field uses a fairly complex format
+ // to encode both the string table ID of the xattr name and the
+ // namespace ID. See the protobuf file for details.
+ int nameId = registerStringId(name);
+ int encodedName = (nameId << XATTR_NAME_OFFSET) |
+ ((nsIdx & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) |
+ (((nsIdx >> 2) & XATTR_NAMESPACE_EXT_MASK)
+ << XATTR_NAMESPACE_EXT_OFFSET);
+ b.setName(encodedName);
+ xattr.verifyNoRemainingKeys("xattr");
+ bld.addXAttrs(b);
+ }
+ xattrs.verifyNoRemainingKeys("xattrs");
+ return bld;
+ }
+
+ private class SecretManagerSectionProcessor implements SectionProcessor {
+ static final String NAME = "SecretManagerSection";
+
+ @Override
+ public void process() throws IOException {
+ Node secretHeader = new Node();
+ loadNodeChildren(secretHeader, "SecretManager fields",
+ "delegationKey", "token");
+ SecretManagerSection.Builder b = SecretManagerSection.newBuilder();
+ Integer currentId = secretHeader.removeChildInt("currentId");
+ if (currentId == null) {
+ throw new IOException("SecretManager section had no <currentId>");
+ }
+ b.setCurrentId(currentId);
+ Integer tokenSequenceNumber = secretHeader.removeChildInt("tokenSequenceNumber");
+ if (tokenSequenceNumber == null) {
+ throw new IOException("SecretManager section had no " +
+ "<tokenSequenceNumber>");
+ }
+ b.setTokenSequenceNumber(tokenSequenceNumber);
+ Integer expectedNumKeys = secretHeader.removeChildInt("numDelegationKeys");
+ if (expectedNumKeys == null) {
+ throw new IOException("SecretManager section had no " +
+ "<numDelegationKeys>");
+ }
+ b.setNumKeys(expectedNumKeys);
+ Integer expectedNumTokens = secretHeader.removeChildInt("numTokens");
+ if (expectedNumTokens == null) {
+ throw new IOException("SecretManager section had no " +
+ "<numTokens>");
+ }
+ b.setNumTokens(expectedNumTokens);
+ secretHeader.verifyNoRemainingKeys("SecretManager");
+ b.build().writeDelimitedTo(out);
+ for (int actualNumKeys = 0; actualNumKeys < expectedNumKeys;
+ actualNumKeys++) {
+ try {
+ expectTag("delegationKey", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + actualNumKeys +
+ " delegation keys out of " + expectedNumKeys, e);
+ }
+ SecretManagerSection.DelegationKey.Builder dbld =
+ SecretManagerSection.DelegationKey.newBuilder();
+ Node dkey = new Node();
+ loadNodeChildren(dkey, "Delegation key fields");
+ Integer id = dkey.removeChildInt("id");
+ if (id == null) {
+ throw new IOException("Delegation key stanza <delegationKey> " +
+ "lacked an <id> field.");
+ }
+ dbld.setId(id);
+ String expiry = dkey.removeChildStr("expiry");
+ if (expiry == null) {
+ throw new IOException("Delegation key stanza <delegationKey> " +
+ "lacked an <expiry> field.");
+ }
+ dbld.setExpiryDate(dateStrToLong(expiry));
+ String keyHex = dkey.removeChildStr("key");
+ if (keyHex == null) {
+ throw new IOException("Delegation key stanza <delegationKey> " +
+ "lacked a <key> field.");
+ }
+ byte[] key = new HexBinaryAdapter().unmarshal(keyHex);
+ dkey.verifyNoRemainingKeys("delegationKey");
+ dbld.setKey(ByteString.copyFrom(key));
+ dbld.build().writeDelimitedTo(out);
+ }
+ for (int actualNumTokens = 0; actualNumTokens < expectedNumTokens;
+ actualNumTokens++) {
+ try {
+ expectTag("token", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + actualNumTokens +
+ " tokens out of " + expectedNumTokens, e);
+ }
+ SecretManagerSection.PersistToken.Builder tbld =
+ SecretManagerSection.PersistToken.newBuilder();
+ Node token = new Node();
+ loadNodeChildren(token, "PersistToken key fields");
+ Integer version = token.removeChildInt("version");
+ if (version != null) {
+ tbld.setVersion(version);
+ }
+ String owner = token.removeChildStr("owner");
+ if (owner != null) {
+ tbld.setOwner(owner);
+ }
+ String renewer = token.removeChildStr("renewer");
+ if (renewer != null) {
+ tbld.setRenewer(renewer);
+ }
+ String realUser = token.removeChildStr("realUser");
+ if (realUser != null) {
+ tbld.setRealUser(realUser);
+ }
+ String issueDateStr = token.removeChildStr("issueDate");
+ if (issueDateStr != null) {
+ tbld.setIssueDate(dateStrToLong(issueDateStr));
+ }
+ String maxDateStr = token.removeChildStr("maxDate");
+ if (maxDateStr != null) {
+ tbld.setMaxDate(dateStrToLong(maxDateStr));
+ }
+ Integer seqNo = token.removeChildInt("sequenceNumber");
+ if (seqNo != null) {
+ tbld.setSequenceNumber(seqNo);
+ }
+ Integer masterKeyId = token.removeChildInt("masterKeyId");
+ if (masterKeyId != null) {
+ tbld.setMasterKeyId(masterKeyId);
+ }
+ String expiryDateStr = token.removeChildStr("expiryDate");
+ if (expiryDateStr != null) {
+ tbld.setExpiryDate(dateStrToLong(expiryDateStr));
+ }
+ token.verifyNoRemainingKeys("token");
+ tbld.build().writeDelimitedTo(out);
+ }
+ expectTagEnd("SecretManagerSection");
+ recordSectionLength(SectionName.SECRET_MANAGER.name());
+ }
+
+ private long dateStrToLong(String dateStr) throws IOException {
+ try {
+ Date date = isoDateFormat.parse(dateStr);
+ return date.getTime();
+ } catch (ParseException e) {
+ throw new IOException("Failed to parse ISO date string " + dateStr, e);
+ }
+ }
+ }
+
+ private class CacheManagerSectionProcessor implements SectionProcessor {
+ static final String NAME = "CacheManagerSection";
+
+ @Override
+ public void process() throws IOException {
+ Node node = new Node();
+ loadNodeChildren(node, "CacheManager fields", "pool", "directive");
+ CacheManagerSection.Builder b = CacheManagerSection.newBuilder();
+ Long nextDirectiveId = node.removeChildLong("nextDirectiveId");
+ if (nextDirectiveId == null) {
+ throw new IOException("CacheManager section had no <nextDirectiveId>");
+ }
+ b.setNextDirectiveId(nextDirectiveId);
+ Integer expectedNumPools = node.removeChildInt("numPools");
+ if (expectedNumPools == null) {
+ throw new IOException("CacheManager section had no <numPools>");
+ }
+ b.setNumPools(expectedNumPools);
+ Integer expectedNumDirectives = node.removeChildInt("numDirectives");
+ if (expectedNumDirectives == null) {
+ throw new IOException("CacheManager section had no <numDirectives>");
+ }
+ b.setNumDirectives(expectedNumDirectives);
+ b.build().writeDelimitedTo(out);
+ long actualNumPools = 0;
+ while (actualNumPools < expectedNumPools) {
+ try {
+ expectTag("pool", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + actualNumPools +
+ " cache pools out of " + expectedNumPools, e);
+ }
+ actualNumPools++;
+ Node pool = new Node();
+ loadNodeChildren(pool, "pool fields", "");
+ processPoolXml(node);
+ }
+ long actualNumDirectives = 0;
+ while (actualNumDirectives < expectedNumDirectives) {
+ try {
+ expectTag("directive", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + actualNumDirectives +
+ " cache pools out of " + expectedNumDirectives, e);
+ }
+ actualNumDirectives++;
+ Node pool = new Node();
+ loadNodeChildren(pool, "directive fields", "");
+ processDirectiveXml(node);
+ }
+ expectTagEnd("CacheManagerSection");
+ recordSectionLength(SectionName.CACHE_MANAGER.name());
+ }
+
+ private void processPoolXml(Node pool) throws IOException {
+ CachePoolInfoProto.Builder bld = CachePoolInfoProto.newBuilder();
+ String poolName = pool.removeChildStr("poolName");
+ if (poolName == null) {
+ throw new IOException("<pool> found without <poolName>");
+ }
+ bld.setPoolName(poolName);
+ String ownerName = pool.removeChildStr("ownerName");
+ if (ownerName == null) {
+ throw new IOException("<pool> found without <ownerName>");
+ }
+ bld.setOwnerName(ownerName);
+ String groupName = pool.removeChildStr("groupName");
+ if (groupName == null) {
+ throw new IOException("<pool> found without <groupName>");
+ }
+ bld.setGroupName(groupName);
+ Integer mode = pool.removeChildInt("mode");
+ if (mode == null) {
+ throw new IOException("<pool> found without <mode>");
+ }
+ bld.setMode(mode);
+ Long limit = pool.removeChildLong("limit");
+ if (limit == null) {
+ throw new IOException("<pool> found without <limit>");
+ }
+ bld.setLimit(limit);
+ Long maxRelativeExpiry = pool.removeChildLong("maxRelativeExpiry");
+ if (maxRelativeExpiry == null) {
+ throw new IOException("<pool> found without <maxRelativeExpiry>");
+ }
+ bld.setMaxRelativeExpiry(maxRelativeExpiry);
+ pool.verifyNoRemainingKeys("pool");
+ bld.build().writeDelimitedTo(out);
+ }
+
+ private void processDirectiveXml(Node directive) throws IOException {
+ CacheDirectiveInfoProto.Builder bld =
+ CacheDirectiveInfoProto.newBuilder();
+ Long id = directive.removeChildLong("id");
+ if (id == null) {
+ throw new IOException("<directive> found without <id>");
+ }
+ bld.setId(id);
+ String path = directive.removeChildStr("path");
+ if (path == null) {
+ throw new IOException("<directive> found without <path>");
+ }
+ bld.setPath(path);
+ Integer replication = directive.removeChildInt("replication");
+ if (replication == null) {
+ throw new IOException("<directive> found without <replication>");
+ }
+ bld.setReplication(replication);
+ String pool = directive.removeChildStr("pool");
+ if (path == null) {
+ throw new IOException("<directive> found without <pool>");
+ }
+ bld.setPool(pool);
+ Node expiration = directive.removeChild("expiration");
+ if (expiration != null) {
+ CacheDirectiveInfoExpirationProto.Builder ebld =
+ CacheDirectiveInfoExpirationProto.newBuilder();
+ Long millis = expiration.removeChildLong("millis");
+ if (millis == null) {
+ throw new IOException("cache directive <expiration> found " +
+ "without <millis>");
+ }
+ ebld.setMillis(millis);
+ if (expiration.removeChildBool("relative")) {
+ ebld.setIsRelative(true);
+ } else {
+ ebld.setIsRelative(false);
+ }
+ bld.setExpiration(ebld);
+ }
+ directive.verifyNoRemainingKeys("directive");
+ bld.build().writeDelimitedTo(out);
+ }
+ }
+
+ private class INodeReferenceSectionProcessor implements SectionProcessor {
+ static final String NAME = "INodeReferenceSection";
+
+ @Override
+ public void process() throws IOException {
+ // There is no header for this section.
+ // We process the repeated <ref> elements.
+ while (true) {
+ XMLEvent ev = expectTag("ref", true);
+ if (ev.isEndElement()) {
+ break;
+ }
+ Node inodeRef = new Node();
+ FsImageProto.INodeReferenceSection.INodeReference.Builder bld =
+ FsImageProto.INodeReferenceSection.INodeReference.newBuilder();
+ loadNodeChildren(inodeRef, "INodeReference");
+ Long referredId = inodeRef.removeChildLong("referredId");
+ if (referredId != null) {
+ bld.setReferredId(referredId);
+ }
+ String name = inodeRef.removeChildStr("name");
+ if (name != null) {
+ bld.setName(ByteString.copyFrom(name, "UTF8"));
+ }
+ Integer dstSnapshotId = inodeRef.removeChildInt("dstSnapshotId");
+ if (dstSnapshotId != null) {
+ bld.setDstSnapshotId(dstSnapshotId);
+ }
+ Integer lastSnapshotId = inodeRef.removeChildInt("lastSnapshotId");
+ if (lastSnapshotId != null) {
+ bld.setLastSnapshotId(lastSnapshotId);
+ }
+ inodeRef.verifyNoRemainingKeys("ref");
+ bld.build().writeDelimitedTo(out);
+ }
+ recordSectionLength(SectionName.INODE_REFERENCE.name());
+ }
+ }
+
+ private class INodeDirectorySectionProcessor implements SectionProcessor {
+ static final String NAME = "INodeDirectorySection";
+
+ @Override
+ public void process() throws IOException {
+ // No header for this section
+ // Process the repeated <directory> elements.
+ while (true) {
+ XMLEvent ev = expectTag("directory", true);
+ if (ev.isEndElement()) {
+ break;
+ }
+ Node directory = new Node();
+ FsImageProto.INodeDirectorySection.DirEntry.Builder bld =
+ FsImageProto.INodeDirectorySection.DirEntry.newBuilder();
+ loadNodeChildren(directory, "directory");
+ Long parent = directory.removeChildLong("parent");
+ if (parent != null) {
+ bld.setParent(parent);
+ }
+ while (true) {
+ Node child = directory.removeChild("child");
+ if (child == null) {
+ break;
+ }
+ bld.addChildren(Long.parseLong(child.getVal()));
+ }
+ while (true) {
+ Node refChild = directory.removeChild("refChild");
+ if (refChild == null) {
+ break;
+ }
+ bld.addRefChildren(Integer.parseInt(refChild.getVal()));
+ }
+ directory.verifyNoRemainingKeys("directory");
+ bld.build().writeDelimitedTo(out);
+ }
+ recordSectionLength(SectionName.INODE_DIR.name());
+ }
+ }
+
+ private class FilesUnderConstructionSectionProcessor
+ implements SectionProcessor {
+ static final String NAME = "FileUnderConstructionSection";
+
+ @Override
+ public void process() throws IOException {
+ // No header for this section type.
+ // Process the repeated files under construction elements.
+ while (true) {
+ XMLEvent ev = expectTag("inode", true);
+ if (ev.isEndElement()) {
+ break;
+ }
+ Node fileUnderConstruction = new Node();
+ loadNodeChildren(fileUnderConstruction, "file under construction");
+ FileUnderConstructionEntry.Builder bld =
+ FileUnderConstructionEntry.newBuilder();
+ Long id = fileUnderConstruction.removeChildLong("id");
+ if (id != null) {
+ bld.setInodeId(id);
+ }
+ String fullpath = fileUnderConstruction.removeChildStr("path");
+ if (fullpath != null) {
+ bld.setFullPath(fullpath);
+ }
+ fileUnderConstruction.verifyNoRemainingKeys("inode");
+ bld.build().writeDelimitedTo(out);
+ }
+ recordSectionLength(SectionName.FILES_UNDERCONSTRUCTION.name());
+ }
+ }
+
+ private class SnapshotSectionProcessor implements SectionProcessor {
+ static final String NAME = "SnapshotSection";
+
+ @Override
+ public void process() throws IOException {
+ FsImageProto.SnapshotSection.Builder bld =
+ FsImageProto.SnapshotSection.newBuilder();
+ Node header = new Node();
+ loadNodeChildren(header, "SnapshotSection fields", "snapshot");
+ Integer snapshotCounter = header.removeChildInt("snapshotCounter");
+ if (snapshotCounter == null) {
+ throw new IOException("No <snapshotCounter> entry found in " +
+ "SnapshotSection header");
+ }
+ bld.setSnapshotCounter(snapshotCounter);
+ Integer expectedNumSnapshots = header.removeChildInt("numSnapshots");
+ if (expectedNumSnapshots == null) {
+ throw new IOException("No <numSnapshots> entry found in " +
+ "SnapshotSection header");
+ }
+ bld.setNumSnapshots(expectedNumSnapshots);
+ while (true) {
+ Node sd = header.removeChild("snapshottableDir");
+ if (sd == null) {
+ break;
+ }
+ Long dir = sd.removeChildLong("dir");
+ sd.verifyNoRemainingKeys("<dir>");
+ bld.addSnapshottableDir(dir);
+ }
+ header.verifyNoRemainingKeys("SnapshotSection");
+ bld.build().writeDelimitedTo(out);
+ int actualNumSnapshots = 0;
+ while (actualNumSnapshots < expectedNumSnapshots) {
+ try {
+ expectTag("snapshot", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + actualNumSnapshots +
+ " <snapshot> entries out of " + expectedNumSnapshots, e);
+ }
+ actualNumSnapshots++;
+ Node snapshot = new Node();
+ loadNodeChildren(snapshot, "snapshot fields");
+ FsImageProto.SnapshotSection.Snapshot.Builder s =
+ FsImageProto.SnapshotSection.Snapshot.newBuilder();
+ Integer snapshotId = snapshot.removeChildInt("id");
+ if (snapshotId == null) {
+ throw new IOException("<snapshot> section was missing <id>");
+ }
+ s.setSnapshotId(snapshotId);
+ Node snapshotRoot = snapshot.removeChild("root");
+ INodeSection.INode.Builder inodeBld = processINodeXml(snapshotRoot);
+ s.setRoot(inodeBld);
+ s.build().writeDelimitedTo(out);
+ }
+ expectTagEnd("SnapshotSection");
+ recordSectionLength(SectionName.SNAPSHOT.name());
+ }
+ }
+
+ private class SnapshotDiffSectionProcessor implements SectionProcessor {
+ static final String NAME = "SnapshotDiffSection";
+
+ @Override
+ public void process() throws IOException {
+ // No header for this section type.
+ LOG.debug("Processing SnapshotDiffSection");
+ while (true) {
+ XMLEvent ev = expectTag("[diff start tag]", true);
+ if (ev.isEndElement()) {
+ String name = ev.asEndElement().getName().getLocalPart();
+ if (name.equals("SnapshotDiffSection")) {
+ break;
+ }
+ throw new IOException("Got unexpected end tag for " + name);
+ }
+ String tagName = ev.asStartElement().getName().getLocalPart();
+ if (tagName.equals("dirDiffEntry")) {
+ processDirDiffEntry();
+ } else if (tagName.equals("fileDiffEntry")) {
+ processFileDiffEntry();
+ } else {
+ throw new IOException("SnapshotDiffSection contained unexpected " +
+ "tag " + tagName);
+ }
+ }
+ recordSectionLength(SectionName.SNAPSHOT_DIFF.name());
+ }
+
+ private void processDirDiffEntry() throws IOException {
+ LOG.debug("Processing dirDiffEntry");
+ DiffEntry.Builder headerBld = DiffEntry.newBuilder();
+ headerBld.setType(DiffEntry.Type.DIRECTORYDIFF);
+ Node dirDiffHeader = new Node();
+ loadNodeChildren(dirDiffHeader, "dirDiffEntry fields", "dirDiff");
+ Long inodeId = dirDiffHeader.removeChildLong("inodeId");
+ if (inodeId == null) {
+ throw new IOException("<dirDiffEntry> contained no <inodeId> entry.");
+ }
+ headerBld.setInodeId(inodeId);
+ Integer expectedDiffs = dirDiffHeader.removeChildInt("count");
+ if (expectedDiffs == null) {
+ throw new IOException("<dirDiffEntry> contained no <count> entry.");
+ }
+ headerBld.setNumOfDiff(expectedDiffs);
+ dirDiffHeader.verifyNoRemainingKeys("dirDiffEntry");
+ headerBld.build().writeDelimitedTo(out);
+ for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) {
+ try {
+ expectTag("dirDiff", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + (actualDiffs + 1) +
+ " diffs out of " + expectedDiffs, e);
+ }
+ Node dirDiff = new Node();
+ loadNodeChildren(dirDiff, "dirDiff fields");
+ FsImageProto.SnapshotDiffSection.DirectoryDiff.Builder bld =
+ FsImageProto.SnapshotDiffSection.DirectoryDiff.newBuilder();
+ Integer snapshotId = dirDiff.removeChildInt("snapshotId");
+ if (snapshotId != null) {
+ bld.setSnapshotId(snapshotId);
+ }
+ Integer childrenSize = dirDiff.removeChildInt("childrenSize");
+ if (childrenSize == null) {
+ throw new IOException("Expected to find <childrenSize> in " +
+ "<dirDiff> section.");
+ }
+ bld.setIsSnapshotRoot(dirDiff.removeChildBool("isSnapshotRoot"));
+ bld.setChildrenSize(childrenSize);
+ String name = dirDiff.removeChildStr("name");
+ if (name != null) {
+ bld.setName(ByteString.copyFrom(name, "UTF8"));
+ }
+ // TODO: add missing snapshotCopy field to XML
+ Integer expectedCreatedListSize =
+ dirDiff.removeChildInt("createdListSize");
+ if (expectedCreatedListSize == null) {
+ throw new IOException("Expected to find <createdListSize> in " +
+ "<dirDiff> section.");
+ }
+ bld.setCreatedListSize(expectedCreatedListSize);
+ while (true) {
+ Node deleted = dirDiff.removeChild("deletedInode");
+ if (deleted == null){
+ break;
+ }
+ bld.addDeletedINode(Long.parseLong(deleted.getVal()));
+ }
+ while (true) {
+ Node deleted = dirDiff.removeChild("deletedInoderef");
+ if (deleted == null){
+ break;
+ }
+ bld.addDeletedINodeRef(Integer.parseInt(deleted.getVal()));
+ }
+ bld.build().writeDelimitedTo(out);
+ // After the DirectoryDiff header comes a list of CreatedListEntry PBs.
+ int actualCreatedListSize = 0;
+ while (true) {
+ Node created = dirDiff.removeChild("created");
+ if (created == null){
+ break;
+ }
+ String cleName = created.removeChildStr("name");
+ if (cleName == null) {
+ throw new IOException("Expected <created> entry to have " +
+ "a <name> field");
+ }
+ created.verifyNoRemainingKeys("created");
+ FsImageProto.SnapshotDiffSection.CreatedListEntry.newBuilder().
+ setName(ByteString.copyFrom(cleName, "UTF8")).
+ build().writeDelimitedTo(out);
+ actualCreatedListSize++;
+ }
+ if (actualCreatedListSize != expectedCreatedListSize) {
+ throw new IOException("<createdListSize> was " +
+ expectedCreatedListSize +", but there were " +
+ actualCreatedListSize + " <created> entries.");
+ }
+ dirDiff.verifyNoRemainingKeys("dirDiff");
+ }
+ expectTagEnd("dirDiffEntry");
+ }
+
+ private void processFileDiffEntry() throws IOException {
+ LOG.debug("Processing fileDiffEntry");
+ DiffEntry.Builder headerBld = DiffEntry.newBuilder();
+ headerBld.setType(DiffEntry.Type.FILEDIFF);
+ Node fileDiffHeader = new Node();
+ loadNodeChildren(fileDiffHeader, "fileDiffEntry fields", "fileDiff");
+ Long inodeId = fileDiffHeader.removeChildLong("inodeid");
+ if (inodeId == null) {
+ throw new IOException("<fileDiffEntry> contained no <inodeid> entry.");
+ }
+ headerBld.setInodeId(inodeId);
+ Integer expectedDiffs = fileDiffHeader.removeChildInt("count");
+ if (expectedDiffs == null) {
+ throw new IOException("<fileDiffEntry> contained no <count> entry.");
+ }
+ headerBld.setNumOfDiff(expectedDiffs);
+ fileDiffHeader.verifyNoRemainingKeys("fileDiffEntry");
+ headerBld.build().writeDelimitedTo(out);
+ for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) {
+ try {
+ expectTag("fileDiff", false);
+ } catch (IOException e) {
+ throw new IOException("Only read " + (actualDiffs + 1) +
+ " diffs out of " + expectedDiffs, e);
+ }
+ Node fileDiff = new Node();
+ loadNodeChildren(fileDiff, "fileDiff fields");
+ FsImageProto.SnapshotDiffSection.FileDiff.Builder bld =
+ FsImageProto.SnapshotDiffSection.FileDiff.newBuilder();
+ Integer snapshotId = fileDiff.removeChildInt("snapshotId");
+ if (snapshotId != null) {
+ bld.setSnapshotId(snapshotId);
+ }
+ Long size = fileDiff.removeChildLong("size");
+ if (size != null) {
+ bld.setFileSize(size);
+ }
+ String name = fileDiff.removeChildStr("name");
+ if (name != null) {
+ bld.setName(ByteString.copyFrom(name, "UTF8"));
+ }
+ // TODO: missing snapshotCopy
+ // TODO: missing blocks
+ fileDiff.verifyNoRemainingKeys("fileDiff");
+ bld.build().writeDelimitedTo(out);
+ }
+ expectTagEnd("fileDiffEntry");
+ }
+ }
+
+ /**
+ * Permission is serialized as a 64-bit long. [0:24):[25:48):[48:64)
+ * (in Big Endian). The first and the second parts are the string ids
+ * of the user and group name, and the last 16 bits are the permission bits.
+ *
+ * @param perm The permission string from the XML.
+ * @return The 64-bit value to use in the fsimage for permission.
+ * @throws IOException If we run out of string IDs in the string table.
+ */
+ private long permissionXmlToU64(String perm) throws IOException {
+ String components[] = perm.split(":");
+ if (components.length != 3) {
+ throw new IOException("Unable to parse permission string " + perm +
+ ": expected 3 components, but only had " + components.length);
+ }
+ String userName = components[0];
+ String groupName = components[1];
+ String modeString = components[2];
+ long userNameId = registerStringId(userName);
+ long groupNameId = registerStringId(groupName);
+ long mode = new FsPermission(modeString).toShort();
+ return (userNameId << 40) | (groupNameId << 16) | mode;
+ }
+
+ /**
+ * The FSImage contains a string table which maps strings to IDs.
+ * This is a simple form of compression which takes advantage of the fact
+ * that the same strings tend to occur over and over again.
+ * This function will return an ID which we can use to represent the given
+ * string. If the string already exists in the string table, we will use
+ * that ID; otherwise, we will allocate a new one.
+ *
+ * @param str The string.
+ * @return The ID in the string table.
+ * @throws IOException If we run out of bits in the string table. We only
+ * have 25 bits.
+ */
+ int registerStringId(String str) throws IOException {
+ Integer id = stringTable.get(str);
+ if (id != null) {
+ return id;
+ }
+ int latestId = latestStringId;
+ if (latestId >= 0x1ffffff) {
+ throw new IOException("Cannot have more than 2**25 " +
+ "strings in the fsimage, because of the limitation on " +
+ "the size of string table IDs.");
+ }
+ stringTable.put(str, latestId);
+ latestStringId++;
+ return latestId;
+ }
+
+ /**
+ * Record the length of a section of the FSImage in our FileSummary object.
+ * The FileSummary appears at the end of the FSImage and acts as a table of
+ * contents for the file.
+ *
+ * @param sectionNamePb The name of the section as it should appear in
+ * the fsimage. (This is different than the XML
+ * name.)
+ * @throws IOException
+ */
+ void recordSectionLength(String sectionNamePb) throws IOException {
+ long curSectionStartOffset = sectionStartOffset;
+ long curPos = out.getCount();
+ //if (sectionNamePb.equals(SectionName.STRING_TABLE.name())) {
+ fileSummaryBld.addSections(FileSummary.Section.newBuilder().
+ setName(sectionNamePb).
+ setLength(curPos - curSectionStartOffset).
+ setOffset(curSectionStartOffset));
+ //}
+ sectionStartOffset = curPos;
+ }
+
+ /**
+ * Read the version tag which starts the XML file.
+ */
+ private void readVersion() throws IOException {
+ try {
+ expectTag("version", false);
+ } catch (IOException e) {
+ // Handle the case where <version> does not exist.
+ // Note: fsimage XML files which are missing <version> are also missing
+ // many other fields that ovi needs to accurately reconstruct the
+ // fsimage.
+ throw new IOException("No <version> section found at the top of " +
+ "the fsimage XML. This XML file is too old to be processed " +
+ "by ovi.", e);
+ }
+ Node version = new Node();
+ loadNodeChildren(version, "version fields");
+ Integer onDiskVersion = version.removeChildInt("onDiskVersion");
+ if (onDiskVersion == null) {
+ throw new IOException("The <version> section doesn't contain " +
+ "the onDiskVersion.");
+ }
+ Integer layoutVersion = version.removeChildInt("layoutVersion");
+ if (layoutVersion == null) {
+ throw new IOException("The <version> section doesn't contain " +
+ "the layoutVersion.");
+ }
+ fileSummaryBld.setOndiskVersion(onDiskVersion);
+ fileSummaryBld.setLayoutVersion(layoutVersion);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loaded <version> with onDiskVersion=" + onDiskVersion +
+ ", layoutVersion=" + layoutVersion + ".");
+ }
+ }
+
+ /**
+ * Write the string table to the fsimage.
+ * @throws IOException
+ */
+ private void writeStringTableSection() throws IOException {
+ FsImageProto.StringTableSection sectionHeader =
+ FsImageProto.StringTableSection.newBuilder().
+ setNumEntry(stringTable.size()).build();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(SectionName.STRING_TABLE.name() + " writing header: {" +
+ TextFormat.printToString(sectionHeader) + "}");
+ }
+ sectionHeader.writeDelimitedTo(out);
+
+ // The entries don't have to be in any particular order, so iterating
+ // over the hash table is fine.
+ for (Map.Entry<String, Integer> entry : stringTable.entrySet()) {
+ FsImageProto.StringTableSection.Entry stEntry =
+ FsImageProto.StringTableSection.Entry.newBuilder().
+ setStr(entry.getKey()).
+ setId(entry.getValue()).
+ build();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Writing string table entry: {" +
+ TextFormat.printToString(stEntry) + "}");
+ }
+ stEntry.writeDelimitedTo(out);
+ }
+ recordSectionLength(SectionName.STRING_TABLE.name());
+ }
+
+ /**
+ * Processes the XML file back into an fsimage.
+ */
+ private void processXml() throws Exception {
+ LOG.debug("Loading <fsimage>.");
+ expectTag("fsimage", false);
+ // Read the <version> tag.
+ readVersion();
+ // Write the HDFSIMG1 magic number which begins the fsimage file.
+ out.write(FSImageUtil.MAGIC_HEADER);
+ // Write a series of fsimage sections.
+ sectionStartOffset = FSImageUtil.MAGIC_HEADER.length;
+ final HashSet<String> unprocessedSections =
+ new HashSet<>(sections.keySet());
+ while (!unprocessedSections.isEmpty()) {
+ XMLEvent ev = expectTag("[section header]", true);
+ if (ev.getEventType() == XMLStreamConstants.END_ELEMENT) {
+ if (ev.asEndElement().getName().getLocalPart().equals("fsimage")) {
+ throw new IOException("FSImage XML ended prematurely, without " +
+ "including section(s) " + StringUtils.join(", ",
+ unprocessedSections));
+ }
+ throw new IOException("Got unexpected tag end event for " +
+ ev.asEndElement().getName().getLocalPart() + " while looking " +
+ "for section header tag.");
+ } else if (ev.getEventType() != XMLStreamConstants.START_ELEMENT) {
+ throw new IOException("Expected section header START_ELEMENT; " +
+ "got event of type " + ev.getEventType());
+ }
+ String sectionName = ev.asStartElement().getName().getLocalPart();
+ if (!unprocessedSections.contains(sectionName)) {
+ throw new IOException("Unknown or duplicate section found for " +
+ sectionName);
+ }
+ SectionProcessor sectionProcessor = sections.get(sectionName);
+ if (sectionProcessor == null) {
+ throw new IOException("Unknown FSImage section " + sectionName +
+ ". Valid section names are [" +
+ StringUtils.join(", ", sections.keySet()) + "]");
+ }
+ unprocessedSections.remove(sectionName);
+ sectionProcessor.process();
+ }
+
+ // Write the StringTable section to disk.
+ // This has to be done after the other sections, since some of them
+ // add entries to the string table.
+ writeStringTableSection();
+
+ // Write the FileSummary section to disk.
+ // This section is always last.
+ long prevOffset = out.getCount();
+ FileSummary fileSummary = fileSummaryBld.build();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Writing FileSummary: {" +
+ TextFormat.printToString(fileSummary) + "}");
+ }
+ // Even though the last 4 bytes of the file gives the FileSummary length,
+ // we still write a varint first that also contains the length.
+ fileSummary.writeDelimitedTo(out);
+
+ // Write the length of the FileSummary section as a fixed-size big
+ // endian 4-byte quantity.
+ int summaryLen = Ints.checkedCast(out.getCount() - prevOffset);
+ byte[] summaryLenBytes = new byte[4];
+ ByteBuffer.wrap(summaryLenBytes).asIntBuffer().put(summaryLen);
+ out.write(summaryLenBytes);
+ }
+
+ /**
+ * Run the OfflineImageReconstructor.
+ *
+ * @param inputPath The input path to use.
+ * @param outputPath The output path to use.
+ *
+ * @throws Exception On error.
+ */
+ public static void run(String inputPath, String outputPath)
+ throws Exception {
+ MessageDigest digester = MD5Hash.getDigester();
+ FileOutputStream fout = null;
+ File foutHash = new File(outputPath + ".md5");
+ Files.deleteIfExists(foutHash.toPath()); // delete any .md5 file that exists
+ CountingOutputStream out = null;
+ FileInputStream fis = null;
+ InputStreamReader reader = null;
+ try {
+ Files.deleteIfExists(Paths.get(outputPath));
+ fout = new FileOutputStream(outputPath);
+ fis = new FileInputStream(inputPath);
+ reader = new InputStreamReader(fis, Charset.forName("UTF-8"));
+ out = new CountingOutputStream(
+ new DigestOutputStream(
+ new BufferedOutputStream(fout), digester));
+ OfflineImageReconstructor oir =
+ new OfflineImageReconstructor(out, reader);
+ oir.processXml();
+ } finally {
+ IOUtils.cleanup(LOG, reader, fis, out, fout);
+ }
+ // Write the md5 file
+ MD5FileUtils.saveMD5File(new File(outputPath),
+ new MD5Hash(digester.digest()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/700b0e40/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index 4afbdb8..e184804 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -60,6 +60,8 @@ public class OfflineImageViewerPB {
+ " * XML: This processor creates an XML document with all elements of\n"
+ " the fsimage enumerated, suitable for further analysis by XML\n"
+ " tools.\n"
+ + " * reverseXML: This processor takes an XML file and creates a\n"
+ + " binary fsimage containing the same elements.\n"
+ " * FileDistribution: This processor analyzes the file size\n"
+ " distribution in the image.\n"
+ " -maxSize specifies the range [0, maxSize] of file sizes to be\n"
@@ -73,15 +75,18 @@ public class OfflineImageViewerPB {
+ " changed via the -delimiter argument.\n"
+ "\n"
+ "Required command line arguments:\n"
- + "-i,--inputFile <arg> FSImage file to process.\n"
+ + "-i,--inputFile <arg> FSImage or XML file to process.\n"
+ "\n"
+ "Optional command line arguments:\n"
+ "-o,--outputFile <arg> Name of output file. If the specified\n"
+ " file exists, it will be overwritten.\n"
+ " (output to stdout by default)\n"
+ + " If the input file was an XML file, we\n"
+ + " will also create an <outputFile>.md5 file.\n"
+ "-p,--processor <arg> Select which type of processor to apply\n"
- + " against image file. (XML|FileDistribution|Web|Delimited)\n"
- + " (Web by default)\n"
+ + " against image file. (XML|FileDistribution|\n"
+ + " ReverseXML|Web|Delimited)\n"
+ + " The default is Web.\n"
+ "-delimiter <arg> Delimiting string to use with Delimited processor. \n"
+ "-t,--temp <arg> Use temporary dir to cache intermediate result to generate\n"
+ " Delimited outputs. If not set, Delimited processor constructs\n"
@@ -177,6 +182,16 @@ public class OfflineImageViewerPB {
new PBImageXmlWriter(conf, out).visit(
new RandomAccessFile(inputFile, "r"));
break;
+ case "ReverseXML":
+ try {
+ OfflineImageReconstructor.run(inputFile, outputFile);
+ } catch (Exception e) {
+ System.err.println("OfflineImageReconstructor failed: " +
+ e.getMessage());
+ e.printStackTrace(System.err);
+ System.exit(1);
+ }
+ break;
case "Web":
String addr = cmd.getOptionValue("addr", "localhost:5978");
try (WebImageViewer viewer = new WebImageViewer(
@@ -200,6 +215,7 @@ public class OfflineImageViewerPB {
System.err.println("Input file ended unexpectedly. Exiting");
} catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage());
+ e.printStackTrace(System.err);
}
return -1;
}
[39/50] [abbrv] hadoop git commit: HDFS-9886. Configuration
properties for hedged read is broken. Contributed by Akira AJISAKA.
Posted by ar...@apache.org.
HDFS-9886. Configuration properties for hedged read is broken. Contributed by Akira AJISAKA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67880cca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67880cca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67880cca
Branch: refs/heads/HDFS-1312
Commit: 67880ccae6568516ff3d13185fd7f250b234a2cf
Parents: 480302b
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Mar 2 10:26:06 2016 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Mar 2 10:26:06 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 ++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67880cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index f036c56..d50c89e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -373,6 +373,8 @@ public interface HdfsClientConfigKeys {
/** dfs.client.hedged.read configuration properties */
interface HedgedRead {
+ String PREFIX = HdfsClientConfigKeys.PREFIX + "hedged.read.";
+
String THRESHOLD_MILLIS_KEY = PREFIX + "threshold.millis";
long THRESHOLD_MILLIS_DEFAULT = 500;
String THREADPOOL_SIZE_KEY = PREFIX + "threadpool.size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67880cca/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9562949..ee204c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2873,6 +2873,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9881. DistributedFileSystem#getTrashRoot returns incorrect path for
encryption zones. (wang)
+ HDFS-9886. Configuration properties for hedged read is broken.
+ (Akira AJISAKA via zhz)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
[44/50] [abbrv] hadoop git commit: HADOOP-12870. Fix typo
admininistration in CommandsManual.md. Contributed by John Zhuge.
Posted by ar...@apache.org.
HADOOP-12870. Fix typo admininistration in CommandsManual.md. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb864d35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb864d35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb864d35
Branch: refs/heads/HDFS-1312
Commit: eb864d35d6c5f0c0af67032d5f6925aa123273af
Parents: f70e317
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 3 18:46:48 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 3 18:46:48 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/CommandsManual.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb864d35/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index dc93beb..3bd3595 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1851,6 +1851,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-9121. InodeTree.java has redundant check for vName while
throwing exception. (Arup Malakar via suresh)
+ HADOOP-12870. Fix typo admininistration in CommandsManual.md.
+ (John Zhuge via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb864d35/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index 58fcbe9..b029e05 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -89,7 +89,7 @@ Many subcommands honor a common set of configuration options to alter their beha
Hadoop Common Commands
======================
-All of these commands are executed from the `hadoop` shell command. They have been broken up into [User Commands](#User_Commands) and [Admininistration Commands](#Admininistration_Commands).
+All of these commands are executed from the `hadoop` shell command. They have been broken up into [User Commands](#User_Commands) and [Administration Commands](#Administration_Commands).
User Commands
-------------
[49/50] [abbrv] hadoop git commit: HDFS-9683. DiskBalancer: Add
cancelPlan implementation. (Contributed by Anu Engineer)
Posted by ar...@apache.org.
HDFS-9683. DiskBalancer: Add cancelPlan implementation. (Contributed by Anu Engineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/889dc8b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/889dc8b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/889dc8b5
Branch: refs/heads/HDFS-1312
Commit: 889dc8b55340064b6499398b3f99f7c65879ce8e
Parents: 3bb9d5e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Mar 3 17:00:52 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Mar 3 17:00:52 2016 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/HDFS-1312_CHANGES.txt | 3 +
.../hadoop/hdfs/server/datanode/DataNode.java | 7 +-
.../hdfs/server/datanode/DiskBalancer.java | 26 +++
.../diskbalancer/DiskBalancerException.java | 3 +-
.../diskbalancer/TestDiskBalancerRPC.java | 213 ++++++++++++-------
5 files changed, 173 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/889dc8b5/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
index 07403cf..919d73e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
@@ -30,3 +30,6 @@ HDFS-1312 Change Log
HDFS-9681. DiskBalancer: Add QueryPlan implementation. (Anu Engineer via
Arpit Agarwal)
+ HDFS-9683. DiskBalancer: Add cancelPlan implementation. (Anu Engineer via
+ Arpit Agarwal)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/889dc8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f440a3a..bfc1972 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -3280,12 +3280,15 @@ public class DataNode extends ReconfigurableBase
this.diskBalancer.submitPlan(planID, planVersion, plan, bandwidth, false);
}
+ /**
+ * Cancels a running plan.
+ * @param planID - Hash string that identifies a plan
+ */
@Override
public void cancelDiskBalancePlan(String planID) throws
IOException {
checkSuperuserPrivilege();
- throw new DiskBalancerException("Not Implemented",
- DiskBalancerException.Result.INTERNAL_ERROR);
+ this.diskBalancer.cancelPlan(planID);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/889dc8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index c01fb4e..81dbb2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -197,6 +197,32 @@ public class DiskBalancer {
}
/**
+ * Cancels a running plan.
+ * @param planID - Hash of the plan to cancel.
+ * @throws DiskBalancerException
+ */
+ public void cancelPlan(String planID) throws DiskBalancerException {
+ lock.lock();
+ try {
+ checkDiskBalancerEnabled();
+ if ((this.planID == null) || (!this.planID.equals(planID))) {
+ LOG.error("Disk Balancer - No such plan. Cancel plan failed. PlanID: " +
+ planID);
+ throw new DiskBalancerException("No such plan.",
+ DiskBalancerException.Result.NO_SUCH_PLAN);
+ }
+ if (!this.future.isDone()) {
+ this.blockMover.setExitFlag();
+ shutdownExecutor();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+
+
+ /**
* Throws if Disk balancer is disabled.
*
* @throws DiskBalancerException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/889dc8b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
index a5e1581..00fe53d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
@@ -35,7 +35,8 @@ public class DiskBalancerException extends IOException {
PLAN_ALREADY_IN_PROGRESS,
INVALID_VOLUME,
INVALID_MOVE,
- INTERNAL_ERROR
+ INTERNAL_ERROR,
+ NO_SUCH_PLAN
}
private final Result result;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/889dc8b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
index 974e973..e29b3b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
@@ -5,9 +5,9 @@
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- * <p/>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -37,6 +37,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import static org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus.Result.NO_PLAN;
import static org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus.Result.PLAN_DONE;
import static org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus.Result.PLAN_UNDER_PROGRESS;
@@ -63,103 +64,163 @@ public class TestDiskBalancerRPC {
}
@Test
- public void testSubmitTestRpc() throws Exception {
- final int dnIndex = 0;
- cluster.restartDataNode(dnIndex);
- cluster.waitActive();
- ClusterConnector nameNodeConnector =
- ConnectorFactory.getCluster(cluster.getFileSystem(0).getUri(), conf);
+ public void testSubmitPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ int planVersion = rpcTestHelper.getPlanVersion();
+ NodePlan plan = rpcTestHelper.getPlan();
+ dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
+ }
- DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster(nameNodeConnector);
- diskBalancerCluster.readClusterInfo();
- Assert.assertEquals(cluster.getDataNodes().size(),
- diskBalancerCluster.getNodes().size());
- diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes());
+ @Test
+ public void testSubmitPlanWithInvalidHash() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ char hashArray[] = planHash.toCharArray();
+ hashArray[0]++;
+ planHash = String.valueOf(hashArray);
+ int planVersion = rpcTestHelper.getPlanVersion();
+ NodePlan plan = rpcTestHelper.getPlan();
+ thrown.expect(DiskBalancerException.class);
+ dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
+ }
- DataNode dataNode = cluster.getDataNodes().get(dnIndex);
- DiskBalancerDataNode node = diskBalancerCluster.getNodeByUUID(
- dataNode.getDatanodeUuid());
- GreedyPlanner planner = new GreedyPlanner(10.0f, node);
- NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort
- ());
- planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
- final int planVersion = 1; // So far we support only one version.
+ @Test
+ public void testSubmitPlanWithInvalidVersion() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ int planVersion = rpcTestHelper.getPlanVersion();
+ planVersion++;
+ NodePlan plan = rpcTestHelper.getPlan();
+ thrown.expect(DiskBalancerException.class);
+ dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
+ }
- String planHash = DigestUtils.sha512Hex(plan.toJson());
+ @Test
+ public void testSubmitPlanWithInvalidPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ int planVersion = rpcTestHelper.getPlanVersion();
+ NodePlan plan = rpcTestHelper.getPlan();
+ thrown.expect(DiskBalancerException.class);
+ dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, "");
+ }
+ @Test
+ public void testCancelPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ int planVersion = rpcTestHelper.getPlanVersion();
+ NodePlan plan = rpcTestHelper.getPlan();
dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
+ dataNode.cancelDiskBalancePlan(planHash);
}
@Test
- public void testCancelTestRpc() throws Exception {
- final int dnIndex = 0;
- cluster.restartDataNode(dnIndex);
- cluster.waitActive();
- ClusterConnector nameNodeConnector =
- ConnectorFactory.getCluster(cluster.getFileSystem(0).getUri(), conf);
-
- DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster(nameNodeConnector);
- diskBalancerCluster.readClusterInfo();
- Assert.assertEquals(cluster.getDataNodes().size(),
- diskBalancerCluster.getNodes().size());
- diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes());
- DiskBalancerDataNode node = diskBalancerCluster.getNodes().get(0);
- GreedyPlanner planner = new GreedyPlanner(10.0f, node);
- NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort
- ());
- planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
-
- final int planVersion = 0; // So far we support only one version.
- DataNode dataNode = cluster.getDataNodes().get(dnIndex);
- String planHash = DigestUtils.sha512Hex(plan.toJson());
-
- // Since submitDiskBalancerPlan is not implemented yet, it throws an
- // Exception, this will be modified with the actual implementation.
- try {
- dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
- } catch (DiskBalancerException ex) {
- // Let us ignore this for time being.
- }
+ public void testCancelNonExistentPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ char hashArray[] = planHash.toCharArray();
+ hashArray[0]++;
+ planHash = String.valueOf(hashArray);
+ NodePlan plan = rpcTestHelper.getPlan();
thrown.expect(DiskBalancerException.class);
dataNode.cancelDiskBalancePlan(planHash);
}
@Test
- public void testQueryTestRpc() throws Exception {
- final int dnIndex = 0;
- cluster.restartDataNode(dnIndex);
- cluster.waitActive();
- ClusterConnector nameNodeConnector =
- ConnectorFactory.getCluster(cluster.getFileSystem(0).getUri(), conf);
-
- DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster
- (nameNodeConnector);
- diskBalancerCluster.readClusterInfo();
- Assert.assertEquals(cluster.getDataNodes().size(),
- diskBalancerCluster.getNodes().size());
- diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes());
- DataNode dataNode = cluster.getDataNodes().get(dnIndex);
- DiskBalancerDataNode node = diskBalancerCluster.getNodeByUUID(
- dataNode.getDatanodeUuid());
- GreedyPlanner planner = new GreedyPlanner(10.0f, node);
- NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort
- ());
- planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
-
- final int planVersion = 1; // So far we support only one version.
- String planHash = DigestUtils.sha512Hex(plan.toJson());
- dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
+ public void testCancelEmptyPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = "";
+ NodePlan plan = rpcTestHelper.getPlan();
+ thrown.expect(DiskBalancerException.class);
+ dataNode.cancelDiskBalancePlan(planHash);
+ }
+
+
+ @Test
+ public void testQueryPlan() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+ String planHash = rpcTestHelper.getPlanHash();
+ int planVersion = rpcTestHelper.getPlanVersion();
+ NodePlan plan = rpcTestHelper.getPlan();
+
+ dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
DiskBalancerWorkStatus status = dataNode.queryDiskBalancerPlan();
Assert.assertTrue(status.getResult() == PLAN_UNDER_PROGRESS ||
status.getResult() == PLAN_DONE);
}
@Test
- public void testgetDiskBalancerSetting() throws Exception {
+ public void testQueryPlanWithoutSubmit() throws Exception {
+ RpcTestHelper rpcTestHelper = new RpcTestHelper().invoke();
+ DataNode dataNode = rpcTestHelper.getDataNode();
+
+ DiskBalancerWorkStatus status = dataNode.queryDiskBalancerPlan();
+ Assert.assertTrue(status.getResult() == NO_PLAN);
+ }
+
+ @Test
+ public void testGetDiskBalancerSetting() throws Exception {
final int dnIndex = 0;
DataNode dataNode = cluster.getDataNodes().get(dnIndex);
thrown.expect(DiskBalancerException.class);
dataNode.getDiskBalancerSetting(
DiskBalancerConstants.DISKBALANCER_BANDWIDTH);
}
+
+ private class RpcTestHelper {
+ private NodePlan plan;
+ private int planVersion;
+ private DataNode dataNode;
+ private String planHash;
+
+ public NodePlan getPlan() {
+ return plan;
+ }
+
+ public int getPlanVersion() {
+ return planVersion;
+ }
+
+ public DataNode getDataNode() {
+ return dataNode;
+ }
+
+ public String getPlanHash() {
+ return planHash;
+ }
+
+ public RpcTestHelper invoke() throws Exception {
+ final int dnIndex = 0;
+ cluster.restartDataNode(dnIndex);
+ cluster.waitActive();
+ ClusterConnector nameNodeConnector =
+ ConnectorFactory.getCluster(cluster.getFileSystem(0).getUri(), conf);
+
+ DiskBalancerCluster diskBalancerCluster =
+ new DiskBalancerCluster(nameNodeConnector);
+ diskBalancerCluster.readClusterInfo();
+ Assert.assertEquals(cluster.getDataNodes().size(),
+ diskBalancerCluster.getNodes().size());
+ diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes());
+ dataNode = cluster.getDataNodes().get(dnIndex);
+ DiskBalancerDataNode node = diskBalancerCluster.getNodeByUUID(
+ dataNode.getDatanodeUuid());
+ GreedyPlanner planner = new GreedyPlanner(10.0f, node);
+ plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
+ planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
+ planVersion = 1;
+ planHash = DigestUtils.sha512Hex(plan.toJson());
+ return this;
+ }
+ }
}
[27/50] [abbrv] hadoop git commit: Add HADOOP-12843 to 2.8.0 in
CHANGES.txt.
Posted by ar...@apache.org.
Add HADOOP-12843 to 2.8.0 in CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44d9bac1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44d9bac1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44d9bac1
Branch: refs/heads/HDFS-1312
Commit: 44d9bac1f58ae64b8a62a197df790e63ec912a72
Parents: d251e55
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Mar 2 00:24:00 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Mar 2 00:24:38 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44d9bac1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 65767f6..b3413cc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1776,6 +1776,8 @@ Release 2.8.0 - UNRELEASED
ProviderUtils.excludeIncompatibleCredentialProviders.
(Larry McCay via cnauroth)
+ HADOOP-12843. Fix findbugs warnings in hadoop-common (branch-2). (aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
[22/50] [abbrv] hadoop git commit: HADOOP-12622. Improve the loggings
in RetryPolicies and RetryInvocationHandler. Contributed by Junping Du
Posted by ar...@apache.org.
HADOOP-12622. Improve the loggings in RetryPolicies and RetryInvocationHandler. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8f390d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8f390d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8f390d0
Branch: refs/heads/HDFS-1312
Commit: d8f390d015510950ccf78174af8891cd613d4438
Parents: 9dafaaa
Author: Jian He <ji...@apache.org>
Authored: Mon Feb 29 16:24:05 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Feb 29 16:24:05 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/io/retry/RetryInvocationHandler.java | 22 ++--
.../apache/hadoop/io/retry/RetryPolicies.java | 60 ++++++++---
.../apache/hadoop/io/retry/TestRetryProxy.java | 101 +++++++++++++++++--
4 files changed, 153 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8f390d0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f444b71..8655d24 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1165,6 +1165,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12825. Log slow name resolutions.
(Sidharta Seethana via stevel)
+ HADOOP-12622. Improve the loggings in RetryPolicies and RetryInvocationHandler.
+ (Junping Du via jianhe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8f390d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 6864d5d..a67c84f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -120,6 +120,7 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
invocationFailoverCount, isIdempotentOrAtMostOnce);
RetryAction failAction = getFailAction(actions);
if (failAction != null) {
+ // fail.
if (failAction.reason != null) {
LOG.warn("Exception while invoking " + currentProxy.proxy.getClass()
+ "." + method.getName() + " over " + currentProxy.proxyInfo
@@ -135,7 +136,8 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
worthLogging |= LOG.isDebugEnabled();
RetryAction failOverAction = getFailOverAction(actions);
long delay = getDelayMillis(actions);
- if (failOverAction != null && worthLogging) {
+
+ if (worthLogging) {
String msg = "Exception while invoking " + method.getName()
+ " of class " + currentProxy.proxy.getClass().getSimpleName()
+ " over " + currentProxy.proxyInfo;
@@ -143,21 +145,21 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
if (invocationFailoverCount > 0) {
msg += " after " + invocationFailoverCount + " fail over attempts";
}
- msg += ". Trying to fail over " + formatSleepMessage(delay);
- LOG.info(msg, ex);
- } else {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Exception while invoking " + method.getName()
- + " of class " + currentProxy.proxy.getClass().getSimpleName()
- + " over " + currentProxy.proxyInfo + ". Retrying "
- + formatSleepMessage(delay), ex);
+
+ if (failOverAction != null) {
+ // failover
+ msg += ". Trying to fail over " + formatSleepMessage(delay);
+ } else {
+ // retry
+ msg += ". Retrying " + formatSleepMessage(delay);
}
+ LOG.info(msg, ex);
}
if (delay > 0) {
Thread.sleep(delay);
}
-
+
if (failOverAction != null) {
// Make sure that concurrent failed method invocations only cause a
// single actual fail over.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8f390d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index 171d52a..131aa8f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,6 +39,8 @@ import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* <p>
* A collection of useful implementations of {@link RetryPolicy}.
@@ -177,10 +179,11 @@ public class RetryPolicies {
@Override
public RetryAction shouldRetry(Exception e, int retries, int failovers,
boolean isIdempotentOrAtMostOnce) throws Exception {
- return RetryAction.FAIL;
+ return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "try once " +
+ "and fail.");
}
}
-
+
static class RetryForever implements RetryPolicy {
@Override
public RetryAction shouldRetry(Exception e, int retries, int failovers,
@@ -221,14 +224,24 @@ public class RetryPolicies {
public RetryAction shouldRetry(Exception e, int retries, int failovers,
boolean isIdempotentOrAtMostOnce) throws Exception {
if (retries >= maxRetries) {
- return RetryAction.FAIL;
+ return new RetryAction(RetryAction.RetryDecision.FAIL, 0 , getReason());
}
return new RetryAction(RetryAction.RetryDecision.RETRY,
- timeUnit.toMillis(calculateSleepTime(retries)));
+ timeUnit.toMillis(calculateSleepTime(retries)), getReason());
}
-
+
+ protected String getReason() {
+ return constructReasonString(maxRetries);
+ }
+
+ @VisibleForTesting
+ public static String constructReasonString(int retries) {
+ return "retries get failed due to exceeded maximum allowed retries " +
+ "number: " + retries;
+ }
+
protected abstract long calculateSleepTime(int retries);
-
+
@Override
public int hashCode() {
return toString().hashCode();
@@ -264,18 +277,37 @@ public class RetryPolicies {
return sleepTime;
}
}
-
- static class RetryUpToMaximumTimeWithFixedSleep extends RetryUpToMaximumCountWithFixedSleep {
- public RetryUpToMaximumTimeWithFixedSleep(long maxTime, long sleepTime, TimeUnit timeUnit) {
+
+ static class RetryUpToMaximumTimeWithFixedSleep extends
+ RetryUpToMaximumCountWithFixedSleep {
+ private long maxTime = 0;
+ private TimeUnit timeUnit;
+
+ public RetryUpToMaximumTimeWithFixedSleep(long maxTime, long sleepTime,
+ TimeUnit timeUnit) {
super((int) (maxTime / sleepTime), sleepTime, timeUnit);
+ this.maxTime = maxTime;
+ this.timeUnit = timeUnit;
+ }
+
+ @Override
+ protected String getReason() {
+ return constructReasonString(this.maxTime, this.timeUnit);
+ }
+
+ @VisibleForTesting
+ public static String constructReasonString(long maxTime,
+ TimeUnit timeUnit) {
+ return "retries get failed due to exceeded maximum allowed time (" +
+ "in " + timeUnit.toString() + "): " + maxTime;
}
}
-
+
static class RetryUpToMaximumCountWithProportionalSleep extends RetryLimited {
public RetryUpToMaximumCountWithProportionalSleep(int maxRetries, long sleepTime, TimeUnit timeUnit) {
super(maxRetries, sleepTime, timeUnit);
}
-
+
@Override
protected long calculateSleepTime(int retries) {
return sleepTime * (retries + 1);
@@ -332,7 +364,8 @@ public class RetryPolicies {
final Pair p = searchPair(curRetry);
if (p == null) {
//no more retries.
- return RetryAction.FAIL;
+ return new RetryAction(RetryAction.RetryDecision.FAIL, 0 , "Retry " +
+ "all pairs in MultipleLinearRandomRetry: " + pairs);
}
//calculate sleep time and return.
@@ -549,6 +582,7 @@ public class RetryPolicies {
protected long calculateSleepTime(int retries) {
return calculateExponentialTime(sleepTime, retries + 1);
}
+
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8f390d0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index 35a45b4..4137dae 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -28,6 +28,15 @@ import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWith
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumTimeWithFixedSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.retryForeverWithFixedSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import static org.junit.Assert.*;
import java.io.IOException;
@@ -41,10 +50,19 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.io.retry.RetryPolicies.RetryUpToMaximumCountWithFixedSleep;
+import org.apache.hadoop.io.retry.RetryPolicies.RetryUpToMaximumTimeWithFixedSleep;
+import org.apache.hadoop.io.retry.RetryPolicies.TryOnceThenFail;
import org.apache.hadoop.io.retry.UnreliableInterface.FatalException;
import org.apache.hadoop.io.retry.UnreliableInterface.UnreliableException;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RemoteException;
+
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
import org.junit.Before;
import org.junit.Test;
@@ -53,25 +71,57 @@ import java.lang.reflect.UndeclaredThrowableException;
public class TestRetryProxy {
private UnreliableImplementation unreliableImpl;
+ private RetryAction caughtRetryAction = null;
@Before
public void setUp() throws Exception {
unreliableImpl = new UnreliableImplementation();
}
+ // answer mockPolicy's method with realPolicy, caught method's return value
+ private void setupMockPolicy(RetryPolicy mockPolicy,
+ final RetryPolicy realPolicy) throws Exception {
+ when(mockPolicy.shouldRetry(any(Exception.class), anyInt(), anyInt(),
+ anyBoolean())).thenAnswer(new Answer<RetryAction>() {
+ @SuppressWarnings("rawtypes")
+ @Override
+ public RetryAction answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ Exception e = (Exception) args[0];
+ int retries = (int) args[1];
+ int failovers = (int) args[2];
+ boolean isIdempotentOrAtMostOnce = (boolean) args[3];
+ caughtRetryAction = realPolicy.shouldRetry(e, retries, failovers,
+ isIdempotentOrAtMostOnce);
+ return caughtRetryAction;
+ }
+ });
+ }
+
@Test
- public void testTryOnceThenFail() throws UnreliableException {
+ public void testTryOnceThenFail() throws Exception {
+ RetryPolicy policy = mock(TryOnceThenFail.class);
+ RetryPolicy realPolicy = TRY_ONCE_THEN_FAIL;
+ setupMockPolicy(policy, realPolicy);
+
UnreliableInterface unreliable = (UnreliableInterface)
- RetryProxy.create(UnreliableInterface.class, unreliableImpl, TRY_ONCE_THEN_FAIL);
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, policy);
unreliable.alwaysSucceeds();
try {
unreliable.failsOnceThenSucceeds();
fail("Should fail");
} catch (UnreliableException e) {
// expected
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ assertEquals("try once and fail.", caughtRetryAction.reason);
+ } catch (Exception e) {
+ fail("Other exception other than UnreliableException should also get " +
+ "failed.");
}
}
-
+
/**
* Test for {@link RetryInvocationHandler#isRpcInvocation(Object)}
*/
@@ -125,25 +175,48 @@ public class TestRetryProxy {
}
@Test
- public void testRetryUpToMaximumCountWithFixedSleep() throws UnreliableException {
+ public void testRetryUpToMaximumCountWithFixedSleep() throws
+ Exception {
+
+ RetryPolicy policy = mock(RetryUpToMaximumCountWithFixedSleep.class);
+ int maxRetries = 8;
+ RetryPolicy realPolicy = retryUpToMaximumCountWithFixedSleep(maxRetries, 1,
+ TimeUnit.NANOSECONDS);
+ setupMockPolicy(policy, realPolicy);
+
UnreliableInterface unreliable = (UnreliableInterface)
- RetryProxy.create(UnreliableInterface.class, unreliableImpl,
- retryUpToMaximumCountWithFixedSleep(8, 1, TimeUnit.NANOSECONDS));
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, policy);
+ // shouldRetry += 1
unreliable.alwaysSucceeds();
+ // shouldRetry += 2
unreliable.failsOnceThenSucceeds();
try {
+ // shouldRetry += (maxRetries -1) (just failed once above)
unreliable.failsTenTimesThenSucceeds();
fail("Should fail");
} catch (UnreliableException e) {
// expected
+ verify(policy, times(maxRetries + 2)).shouldRetry(any(Exception.class),
+ anyInt(), anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ assertEquals(RetryUpToMaximumCountWithFixedSleep.constructReasonString(
+ maxRetries), caughtRetryAction.reason);
+ } catch (Exception e) {
+ fail("Other exception other than UnreliableException should also get " +
+ "failed.");
}
}
@Test
- public void testRetryUpToMaximumTimeWithFixedSleep() throws UnreliableException {
+ public void testRetryUpToMaximumTimeWithFixedSleep() throws Exception {
+ RetryPolicy policy = mock(RetryUpToMaximumTimeWithFixedSleep.class);
+ long maxTime = 80L;
+ RetryPolicy realPolicy = retryUpToMaximumTimeWithFixedSleep(maxTime, 10,
+ TimeUnit.NANOSECONDS);
+ setupMockPolicy(policy, realPolicy);
+
UnreliableInterface unreliable = (UnreliableInterface)
- RetryProxy.create(UnreliableInterface.class, unreliableImpl,
- retryUpToMaximumTimeWithFixedSleep(80, 10, TimeUnit.NANOSECONDS));
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl, policy);
unreliable.alwaysSucceeds();
unreliable.failsOnceThenSucceeds();
try {
@@ -151,9 +224,17 @@ public class TestRetryProxy {
fail("Should fail");
} catch (UnreliableException e) {
// expected
+ verify(policy, times((int)(maxTime/10) + 2)).shouldRetry(any(Exception.class),
+ anyInt(), anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ assertEquals(RetryUpToMaximumTimeWithFixedSleep.constructReasonString(
+ maxTime, TimeUnit.NANOSECONDS), caughtRetryAction.reason);
+ } catch (Exception e) {
+ fail("Other exception other than UnreliableException should also get " +
+ "failed.");
}
}
-
+
@Test
public void testRetryUpToMaximumCountWithProportionalSleep() throws UnreliableException {
UnreliableInterface unreliable = (UnreliableInterface)
[35/50] [abbrv] hadoop git commit: HDFS-9766.
TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently. Contributed
by Xiao Chen.
Posted by ar...@apache.org.
HDFS-9766. TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2ddf824
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2ddf824
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2ddf824
Branch: refs/heads/HDFS-1312
Commit: e2ddf824694eb4605f3bb04a9c26e4b98529f5bc
Parents: f2ba7da
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Mar 2 14:13:43 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Mar 2 14:17:35 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/datanode/TestDataNodeMetrics.java | 48 ++++++++++++--------
2 files changed, 31 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2ddf824/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 421ee68..a87ffe0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2971,6 +2971,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9880. TestDatanodeRegistration fails occasionally (kihwal)
+ HDFS-9766. TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently.
+ (Xiao Chen via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2ddf824/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 5c0a971..5f9b602 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -26,8 +26,10 @@ import static org.junit.Assert.*;
import java.io.Closeable;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.List;
+import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
@@ -47,6 +49,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.Test;
import org.mockito.Mockito;
@@ -255,38 +258,43 @@ public class TestDataNodeMetrics {
* and reading causes totalReadTime to move.
* @throws Exception
*/
- @Test
+ @Test(timeout=60000)
public void testDataNodeTimeSpend() throws Exception {
Configuration conf = new HdfsConfiguration();
SimulatedFSDataset.setFactory(conf);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
try {
- FileSystem fs = cluster.getFileSystem();
+ final FileSystem fs = cluster.getFileSystem();
List<DataNode> datanodes = cluster.getDataNodes();
assertEquals(datanodes.size(), 1);
- DataNode datanode = datanodes.get(0);
+ final DataNode datanode = datanodes.get(0);
MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
final long LONG_FILE_LEN = 1024 * 1024 * 10;
- long startWriteValue = getLongCounter("TotalWriteTime", rb);
- long startReadValue = getLongCounter("TotalReadTime", rb);
-
- for (int x =0; x < 50; x++) {
- DFSTestUtil.createFile(fs, new Path("/time.txt."+ x),
- LONG_FILE_LEN, (short) 1, Time.monotonicNow());
- }
-
- for (int x =0; x < 50; x++) {
- DFSTestUtil.readFile(fs, new Path("/time.txt." + x));
- }
+ final long startWriteValue = getLongCounter("TotalWriteTime", rb);
+ final long startReadValue = getLongCounter("TotalReadTime", rb);
+ final AtomicInteger x = new AtomicInteger(0);
- MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
- long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
- long endReadValue = getLongCounter("TotalReadTime", rbNew);
// Lets Metric system update latest metrics
- Thread.sleep(100);
- assertTrue(endReadValue > startReadValue);
- assertTrue(endWriteValue > startWriteValue);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ x.getAndIncrement();
+ try {
+ DFSTestUtil.createFile(fs, new Path("/time.txt." + x.get()),
+ LONG_FILE_LEN, (short) 1, Time.monotonicNow());
+ DFSTestUtil.readFile(fs, new Path("/time.txt." + x.get()));
+ } catch (IOException ioe) {
+ LOG.error("Caught IOException while ingesting DN metrics", ioe);
+ return false;
+ }
+ MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
+ final long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
+ final long endReadValue = getLongCounter("TotalReadTime", rbNew);
+ return endWriteValue > startWriteValue
+ && endReadValue > startReadValue;
+ }
+ }, 30, 30000);
} finally {
if (cluster != null) {
cluster.shutdown();
[07/50] [abbrv] hadoop git commit: YARN-4718. Rename variables in
SchedulerNode to reduce ambiguity post YARN-1011. (Inigo Goiri via kasha)
Posted by ar...@apache.org.
YARN-4718. Rename variables in SchedulerNode to reduce ambiguity post YARN-1011. (Inigo Goiri via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9692770
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9692770
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9692770
Branch: refs/heads/HDFS-1312
Commit: f9692770a58af0ab082eb7f15da9cbdcd177605b
Parents: 7545ce6
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Sun Feb 28 09:35:59 2016 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Sun Feb 28 09:35:59 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/SchedulerNode.java | 172 ++++++++++------
.../scheduler/SchedulerNodeReport.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 10 +-
.../scheduler/capacity/ParentQueue.java | 2 +-
.../allocator/IncreaseContainerAllocator.java | 4 +-
.../allocator/RegularContainerAllocator.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 6 +-
.../scheduler/fair/FairScheduler.java | 8 +-
.../scheduler/fifo/FifoScheduler.java | 20 +-
.../TestWorkPreservingRMRestart.java | 12 +-
.../scheduler/TestAbstractYarnScheduler.java | 4 +-
.../capacity/TestCapacityScheduler.java | 4 +-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 4 +-
.../capacity/TestContainerResizing.java | 2 +-
.../capacity/TestIncreaseAllocationExpirer.java | 2 +-
.../scheduler/capacity/TestLeafQueue.java | 16 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestReservations.java | 196 +++++++++----------
.../scheduler/capacity/TestUtils.java | 2 +-
.../scheduler/fair/TestFSAppAttempt.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 6 +-
25 files changed, 273 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c27d191..d2f871a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -146,6 +146,9 @@ Release 2.9.0 - UNRELEASED
YARN-4697. NM aggregation thread pool is not bound by
limits (haibochen via rkanter)
+ YARN-4718. Rename variables in SchedulerNode to reduce ambiguity post
+ YARN-1011. (Inigo Goiri via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 03edd40..33ab2f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -54,9 +54,9 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
- private Resource availableResource = Resource.newInstance(0, 0);
- private Resource usedResource = Resource.newInstance(0, 0);
- private Resource totalResourceCapability;
+ private Resource unallocatedResource = Resource.newInstance(0, 0);
+ private Resource allocatedResource = Resource.newInstance(0, 0);
+ private Resource totalResource;
private RMContainer reservedContainer;
private volatile int numContainers;
private volatile ResourceUtilization containersUtilization =
@@ -65,20 +65,20 @@ public abstract class SchedulerNode {
ResourceUtilization.newInstance(0, 0, 0f);
- /* set of containers that are allocated containers */
+ /** Set of containers that are allocated containers. */
private final Map<ContainerId, RMContainer> launchedContainers =
- new HashMap<ContainerId, RMContainer>();
+ new HashMap<>();
private final RMNode rmNode;
private final String nodeName;
-
+
private volatile Set<String> labels = null;
-
+
public SchedulerNode(RMNode node, boolean usePortForNodeName,
Set<String> labels) {
this.rmNode = node;
- this.availableResource = Resources.clone(node.getTotalCapability());
- this.totalResourceCapability = Resources.clone(node.getTotalCapability());
+ this.unallocatedResource = Resources.clone(node.getTotalCapability());
+ this.totalResource = Resources.clone(node.getTotalCapability());
if (usePortForNodeName) {
nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
} else {
@@ -97,23 +97,26 @@ public abstract class SchedulerNode {
/**
* Set total resources on the node.
- * @param resource total resources on the node.
+ * @param resource Total resources on the node.
*/
public synchronized void setTotalResource(Resource resource){
- this.totalResourceCapability = resource;
- this.availableResource = Resources.subtract(totalResourceCapability,
- this.usedResource);
+ this.totalResource = resource;
+ this.unallocatedResource = Resources.subtract(totalResource,
+ this.allocatedResource);
}
/**
* Get the ID of the node which contains both its hostname and port.
- *
- * @return the ID of the node
+ * @return The ID of the node.
*/
public NodeId getNodeID() {
return this.rmNode.getNodeID();
}
+ /**
+ * Get HTTP address for the node.
+ * @return HTTP address for the node.
+ */
public String getHttpAddress() {
return this.rmNode.getHttpAddress();
}
@@ -126,8 +129,7 @@ public abstract class SchedulerNode {
* {@link YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} constant.
* The main usecase of this is Yarn minicluster to be able to differentiate
* node manager instances by their port number.
- *
- * @return name of the node for scheduling matching decisions.
+ * @return Name of the node for scheduling matching decisions.
*/
public String getNodeName() {
return nodeName;
@@ -135,7 +137,6 @@ public abstract class SchedulerNode {
/**
* Get rackname.
- *
* @return rackname
*/
public String getRackName() {
@@ -145,13 +146,11 @@ public abstract class SchedulerNode {
/**
* The Scheduler has allocated containers on this node to the given
* application.
- *
- * @param rmContainer
- * allocated container
+ * @param rmContainer Allocated container
*/
public synchronized void allocateContainer(RMContainer rmContainer) {
Container container = rmContainer.getContainer();
- deductAvailableResource(container.getResource());
+ deductUnallocatedResource(container.getResource());
++numContainers;
launchedContainers.put(container.getId(), rmContainer);
@@ -159,27 +158,35 @@ public abstract class SchedulerNode {
LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + rmNode.getNodeAddress()
+ ", which has " + numContainers + " containers, "
- + getUsedResource() + " used and " + getAvailableResource()
+ + getAllocatedResource() + " used and " + getUnallocatedResource()
+ " available after allocation");
}
-
+
+ /**
+ * Change the resources allocated for a container.
+ * @param containerId Identifier of the container to change.
+ * @param deltaResource Change in the resource allocation.
+ * @param increase True if the change is an increase of allocation.
+ */
private synchronized void changeContainerResource(ContainerId containerId,
Resource deltaResource, boolean increase) {
if (increase) {
- deductAvailableResource(deltaResource);
+ deductUnallocatedResource(deltaResource);
} else {
- addAvailableResource(deltaResource);
+ addUnallocatedResource(deltaResource);
}
LOG.info((increase ? "Increased" : "Decreased") + " container "
+ containerId + " of capacity " + deltaResource + " on host "
+ rmNode.getNodeAddress() + ", which has " + numContainers
- + " containers, " + getUsedResource() + " used and "
- + getAvailableResource() + " available after allocation");
+ + " containers, " + getAllocatedResource() + " used and "
+ + getUnallocatedResource() + " available after allocation");
}
/**
- * The Scheduler increased container
+ * Increase the resources allocated to a container.
+ * @param containerId Identifier of the container to change.
+ * @param deltaResource Increase of resource allocation.
*/
public synchronized void increaseContainer(ContainerId containerId,
Resource deltaResource) {
@@ -187,7 +194,9 @@ public abstract class SchedulerNode {
}
/**
- * The Scheduler decreased container
+ * Decrease the resources allocated to a container.
+ * @param containerId Identifier of the container to change.
+ * @param deltaResource Decrease of resource allocation.
*/
public synchronized void decreaseContainer(ContainerId containerId,
Resource deltaResource) {
@@ -195,32 +204,33 @@ public abstract class SchedulerNode {
}
/**
- * Get available resources on the node.
- *
- * @return available resources on the node
+ * Get unallocated resources on the node.
+ * @return Unallocated resources on the node
*/
- public synchronized Resource getAvailableResource() {
- return this.availableResource;
+ public synchronized Resource getUnallocatedResource() {
+ return this.unallocatedResource;
}
/**
- * Get used resources on the node.
- *
- * @return used resources on the node
+ * Get allocated resources on the node.
+ * @return Allocated resources on the node
*/
- public synchronized Resource getUsedResource() {
- return this.usedResource;
+ public synchronized Resource getAllocatedResource() {
+ return this.allocatedResource;
}
/**
* Get total resources on the node.
- *
- * @return total resources on the node.
+ * @return Total resources on the node.
*/
public synchronized Resource getTotalResource() {
- return this.totalResourceCapability;
+ return this.totalResource;
}
+ /**
+ * Check if a container is launched by this node.
+ * @return If the container is launched by the node.
+ */
public synchronized boolean isValidContainer(ContainerId containerId) {
if (launchedContainers.containsKey(containerId)) {
return true;
@@ -228,16 +238,18 @@ public abstract class SchedulerNode {
return false;
}
+ /**
+ * Update the resources of the node when allocating a new container.
+ * @param container Container to allocate.
+ */
private synchronized void updateResource(Container container) {
- addAvailableResource(container.getResource());
+ addUnallocatedResource(container.getResource());
--numContainers;
}
/**
* Release an allocated container on this node.
- *
- * @param container
- * container to be released
+ * @param container Container to be released.
*/
public synchronized void releaseContainer(Container container) {
if (!isValidContainer(container.getId())) {
@@ -245,7 +257,7 @@ public abstract class SchedulerNode {
return;
}
- /* remove the containers from the nodemanger */
+ // Remove the containers from the nodemanger
if (null != launchedContainers.remove(container.getId())) {
updateResource(container);
}
@@ -253,81 +265,118 @@ public abstract class SchedulerNode {
LOG.info("Released container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + rmNode.getNodeAddress()
+ ", which currently has " + numContainers + " containers, "
- + getUsedResource() + " used and " + getAvailableResource()
+ + getAllocatedResource() + " used and " + getUnallocatedResource()
+ " available" + ", release resources=" + true);
}
- private synchronized void addAvailableResource(Resource resource) {
+ /**
+ * Add unallocated resources to the node. This is used when unallocating a
+ * container.
+ * @param resource Resources to add.
+ */
+ private synchronized void addUnallocatedResource(Resource resource) {
if (resource == null) {
LOG.error("Invalid resource addition of null resource for "
+ rmNode.getNodeAddress());
return;
}
- Resources.addTo(availableResource, resource);
- Resources.subtractFrom(usedResource, resource);
+ Resources.addTo(unallocatedResource, resource);
+ Resources.subtractFrom(allocatedResource, resource);
}
- private synchronized void deductAvailableResource(Resource resource) {
+ /**
+ * Deduct unallocated resources from the node. This is used when allocating a
+ * container.
+ * @param resource Resources to deduct.
+ */
+ private synchronized void deductUnallocatedResource(Resource resource) {
if (resource == null) {
LOG.error("Invalid deduction of null resource for "
+ rmNode.getNodeAddress());
return;
}
- Resources.subtractFrom(availableResource, resource);
- Resources.addTo(usedResource, resource);
+ Resources.subtractFrom(unallocatedResource, resource);
+ Resources.addTo(allocatedResource, resource);
}
/**
* Reserve container for the attempt on this node.
+ * @param attempt Application attempt asking for the reservation.
+ * @param priority Priority of the reservation.
+ * @param container Container reserving resources for.
*/
public abstract void reserveResource(SchedulerApplicationAttempt attempt,
Priority priority, RMContainer container);
/**
* Unreserve resources on this node.
+ * @param attempt Application attempt that had done the reservation.
*/
public abstract void unreserveResource(SchedulerApplicationAttempt attempt);
@Override
public String toString() {
return "host: " + rmNode.getNodeAddress() + " #containers="
- + getNumContainers() + " available=" + getAvailableResource()
- + " used=" + getUsedResource();
+ + getNumContainers() + " available=" + getUnallocatedResource()
+ + " used=" + getAllocatedResource();
}
/**
* Get number of active containers on the node.
- *
- * @return number of active containers on the node
+ * @return Number of active containers on the node.
*/
public int getNumContainers() {
return numContainers;
}
+ /**
+ * Get the running containers in the node.
+ * @return List of running containers in the node.
+ */
public synchronized List<RMContainer> getRunningContainers() {
return new ArrayList<RMContainer>(launchedContainers.values());
}
+ /**
+ * Get the reserved container in the node.
+ * @return Reserved container in the node.
+ */
public synchronized RMContainer getReservedContainer() {
return reservedContainer;
}
+ /**
+ * Set the reserved container in the node.
+ * @param reservedContainer Reserved container in the node.
+ */
protected synchronized void
setReservedContainer(RMContainer reservedContainer) {
this.reservedContainer = reservedContainer;
}
+ /**
+ * Recover a container.
+ * @param rmContainer Container to recover.
+ */
public synchronized void recoverContainer(RMContainer rmContainer) {
if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
return;
}
allocateContainer(rmContainer);
}
-
+
+ /**
+ * Get the labels for the node.
+ * @return Set of labels for the node.
+ */
public Set<String> getLabels() {
return labels;
}
-
+
+ /**
+ * Update the labels for the node.
+ * @param labels Set of labels for the node.
+ */
public void updateLabels(Set<String> labels) {
this.labels = labels;
}
@@ -336,6 +385,7 @@ public abstract class SchedulerNode {
* Get partition of which the node belongs to, if node-labels of this node is
* empty or null, it belongs to NO_LABEL partition. And since we only support
* one partition for each node (YARN-2694), first label will be its partition.
+ * @return Partition for the node.
*/
public String getPartition() {
if (this.labels == null || this.labels.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index b1f6c64..fa71a25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -33,8 +33,8 @@ public class SchedulerNodeReport {
private final int num;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getUsedResource();
- this.avail = node.getAvailableResource();
+ this.used = node.getAllocatedResource();
+ this.avail = node.getUnallocatedResource();
this.num = node.getNumContainers();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 7f844a0..acc2782 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1096,7 +1096,7 @@ public class CapacityScheduler extends
.handle(
new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(nm.getNodeID())
- .getUsedResource(), 0)));
+ .getAllocatedResource(), 0)));
}
schedulerHealth.updateSchedulerReleaseDetails(lastNodeUpdateTime,
releaseResources);
@@ -1109,8 +1109,8 @@ public class CapacityScheduler extends
// Now node data structures are upto date and ready for scheduling.
if(LOG.isDebugEnabled()) {
- LOG.debug("Node being looked for scheduling " + nm
- + " availableResource: " + node.getAvailableResource());
+ LOG.debug("Node being looked for scheduling " + nm +
+ " availableResource: " + node.getUnallocatedResource());
}
}
@@ -1254,11 +1254,11 @@ public class CapacityScheduler extends
// Try to schedule more if there are no reservations to fulfill
if (node.getReservedContainer() == null) {
- if (calculator.computeAvailableContainers(node.getAvailableResource(),
+ if (calculator.computeAvailableContainers(node.getUnallocatedResource(),
minimumAllocation) > 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to schedule on node: " + node.getNodeName() +
- ", available: " + node.getAvailableResource());
+ ", available: " + node.getUnallocatedResource());
}
assignment = root.assignContainers(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index a7d8796..7cf5565 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -496,7 +496,7 @@ public class ParentQueue extends AbstractCSQueue {
private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) {
return (node.getReservedContainer() == null) &&
Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
- node.getAvailableResource(), minimumAllocation);
+ node.getUnallocatedResource(), minimumAllocation);
}
private ResourceLimits getResourceLimitsOfChild(CSQueue child,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
index 16cf6d3..25e5824 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
@@ -112,7 +112,7 @@ public class IncreaseContainerAllocator extends AbstractContainerAllocator {
SchedulerNode node, Resource cluster,
SchedContainerChangeRequest increaseRequest) {
if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
- node.getAvailableResource())) {
+ node.getUnallocatedResource())) {
// OK, we can allocate this increase request
// Unreserve it first
application.unreserve(increaseRequest.getPriority(),
@@ -141,7 +141,7 @@ public class IncreaseContainerAllocator extends AbstractContainerAllocator {
private CSAssignment allocateIncreaseRequest(FiCaSchedulerNode node,
Resource cluster, SchedContainerChangeRequest increaseRequest) {
if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
- node.getAvailableResource())) {
+ node.getUnallocatedResource())) {
// Notify node
node.increaseContainer(increaseRequest.getContainerId(),
increaseRequest.getDeltaCapacity());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 820cccd..e168edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -423,7 +423,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
Resource capability = request.getCapability();
- Resource available = node.getAvailableResource();
+ Resource available = node.getUnallocatedResource();
Resource totalResource = node.getTotalResource();
if (!Resources.lessThanOrEqual(rc, clusterResource,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 730743c..4d563cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -589,7 +589,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
diagnosticMessageBldr.append(", Total resource : ");
diagnosticMessageBldr.append(node.getTotalResource());
diagnosticMessageBldr.append(", Available resource : ");
- diagnosticMessageBldr.append(node.getAvailableResource());
+ diagnosticMessageBldr.append(node.getUnallocatedResource());
diagnosticMessageBldr.append(" ).");
updateAMContainerDiagnostics(AMState.ACTIVATED, diagnosticMessageBldr.toString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 7e0a693..f1cefad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -191,7 +191,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
SchedulerNode node = scheduler.getSchedulerNode(nodeId);
if (node != null) {
Resources.subtractFrom(availableResources,
- node.getAvailableResource());
+ node.getUnallocatedResource());
}
}
if (availableResources.getMemory() < 0) {
@@ -613,7 +613,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource capability = request.getCapability();
// How much does the node have?
- Resource available = node.getAvailableResource();
+ Resource available = node.getUnallocatedResource();
Container container = null;
if (reserved) {
@@ -840,7 +840,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Note that we have an assumption here that
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
- node.getAvailableResource())) {
+ node.getUnallocatedResource())) {
assignContainer(node, true);
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 1dbcda2..2801bee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1069,7 +1069,7 @@ public class FairScheduler extends
.handle(
new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(nm.getNodeID())
- .getUsedResource(), 0)));
+ .getAllocatedResource(), 0)));
}
if (continuousSchedulingEnabled) {
@@ -1105,7 +1105,7 @@ public class FairScheduler extends
FSSchedulerNode node = getFSSchedulerNode(nodeId);
try {
if (node != null && Resources.fitsIn(minimumAllocation,
- node.getAvailableResource())) {
+ node.getUnallocatedResource())) {
attemptScheduling(node);
}
} catch (Throwable ex) {
@@ -1137,8 +1137,8 @@ public class FairScheduler extends
return -1;
}
return RESOURCE_CALCULATOR.compare(clusterResource,
- nodes.get(n2).getAvailableResource(),
- nodes.get(n1).getAvailableResource());
+ nodes.get(n2).getUnallocatedResource(),
+ nodes.get(n1).getUnallocatedResource());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index a0e14111..147c3f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -527,7 +527,7 @@ public class FifoScheduler extends
// Done
if (Resources.lessThan(resourceCalculator, clusterResource,
- node.getAvailableResource(), minimumAllocation)) {
+ node.getUnallocatedResource(), minimumAllocation)) {
break;
}
}
@@ -682,13 +682,9 @@ public class FifoScheduler extends
" request=" + request + " type=" + type);
Resource capability = request.getCapability();
- int availableContainers =
- node.getAvailableResource().getMemory() / capability.getMemory(); // TODO: A buggy
- // application
- // with this
- // zero would
- // crash the
- // scheduler.
+ // TODO: A buggy application with this zero would crash the scheduler.
+ int availableContainers = node.getUnallocatedResource().getMemory() /
+ capability.getMemory();
int assignedContainers =
Math.min(assignableContainers, availableContainers);
@@ -760,7 +756,7 @@ public class FifoScheduler extends
.handle(
new RMNodeResourceUpdateEvent(rmNode.getNodeID(), ResourceOption
.newInstance(getSchedulerNode(rmNode.getNodeID())
- .getUsedResource(), 0)));
+ .getAllocatedResource(), 0)));
}
if (rmContext.isWorkPreservingRecoveryEnabled()
@@ -769,14 +765,14 @@ public class FifoScheduler extends
}
if (Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
- node.getAvailableResource(),minimumAllocation)) {
+ node.getUnallocatedResource(), minimumAllocation)) {
LOG.debug("Node heartbeat " + rmNode.getNodeID() +
- " available resource = " + node.getAvailableResource());
+ " available resource = " + node.getUnallocatedResource());
assignContainers(node);
LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = "
- + node.getAvailableResource());
+ + node.getUnallocatedResource());
}
updateAvailableResourcesMetrics();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index f0aa470..b32dfbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -213,11 +213,11 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertTrue(
"SchedulerNode#toString is not in expected format",
schedulerNode1
- .toString().contains(schedulerNode1.getAvailableResource().toString()));
+ .toString().contains(schedulerNode1.getUnallocatedResource().toString()));
assertTrue(
"SchedulerNode#toString is not in expected format",
schedulerNode1
- .toString().contains(schedulerNode1.getUsedResource().toString()));
+ .toString().contains(schedulerNode1.getAllocatedResource().toString()));
// ********* check scheduler node state.*******
// 2 running containers.
@@ -234,8 +234,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertEquals(2, schedulerNode1.getNumContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
- schedulerNode1.getAvailableResource());
- assertEquals(usedResources, schedulerNode1.getUsedResource());
+ schedulerNode1.getUnallocatedResource());
+ assertEquals(usedResources, schedulerNode1.getAllocatedResource());
Resource availableResources = Resources.subtract(nmResource, usedResources);
// ***** check queue state based on the underlying scheduler ********
@@ -379,8 +379,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertEquals(2, schedulerNode1.getNumContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
- schedulerNode1.getAvailableResource());
- assertEquals(usedResources, schedulerNode1.getUsedResource());
+ schedulerNode1.getUnallocatedResource());
+ assertEquals(usedResources, schedulerNode1.getAllocatedResource());
Resource availableResources = Resources.subtract(nmResource, usedResources);
// 6. Verify the scheduler state like attempt info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 05c897f..8411a4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -268,12 +268,12 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
SchedulerNode mockNode1 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
- when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
+ when(mockNode1.getUnallocatedResource()).thenReturn(emptyResource);
when(mockNode1.getTotalResource()).thenReturn(fullResource1);
SchedulerNode mockNode2 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
- when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
+ when(mockNode2.getUnallocatedResource()).thenReturn(emptyResource);
when(mockNode2.getTotalResource()).thenReturn(fullResource2);
verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index bd2c4fe..78ed0a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3431,7 +3431,7 @@ public class TestCapacityScheduler {
Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getUsedResource();
+ .getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 1 * GB);
Assert.assertEquals(usedResource.getVirtualCores(), 1);
// Check total resource of scheduler node is also changed to 1 GB 1 core
@@ -3443,7 +3443,7 @@ public class TestCapacityScheduler {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
+ .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 88c7c13..5169337 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -147,9 +147,9 @@ public class TestChildQueueOrder {
any(ResourceLimits.class), any(SchedulingMode.class));
// Mock the node's resource availability
- Resource available = node.getAvailableResource();
+ Resource available = node.getUnallocatedResource();
doReturn(Resources.subtractFrom(available, allocatedResource)).
- when(node).getAvailableResource();
+ when(node).getUnallocatedResource();
}
return new CSAssignment(allocatedResource, type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b5b2222..1f22a06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -388,7 +388,7 @@ public class TestContainerAllocation {
// NM1 has available resource = 2G (8G - 2 * 1G - 4G)
Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
- .getAvailableResource().getMemory());
+ .getUnallocatedResource().getMemory());
Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
// Usage of queue = 4G + 2 * 1G + 4G (reserved)
Assert.assertEquals(10 * GB, cs.getRootQueue().getQueueResourceUsage()
@@ -401,7 +401,7 @@ public class TestContainerAllocation {
// App2's reservation will be cancelled
Assert.assertTrue(schedulerApp2.getReservedContainers().size() == 0);
Assert.assertEquals(2 * GB, cs.getNode(nm1.getNodeId())
- .getAvailableResource().getMemory());
+ .getUnallocatedResource().getMemory());
Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
Assert.assertEquals(6 * GB, cs.getRootQueue().getQueueResourceUsage()
.getUsed().getMemory());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 9e29842..f04748d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -1121,6 +1121,6 @@ public class TestContainerResizing {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
SchedulerNode node = cs.getNode(nodeId);
Assert
- .assertEquals(expectedMemory, node.getAvailableResource().getMemory());
+ .assertEquals(expectedMemory, node.getUnallocatedResource().getMemory());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index d7ac0b2..645086d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -429,7 +429,7 @@ public class TestIncreaseAllocationExpirer {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
SchedulerNode node = cs.getNode(nodeId);
Assert
- .assertEquals(expectedMemory, node.getAvailableResource().getMemory());
+ .assertEquals(expectedMemory, node.getUnallocatedResource().getMemory());
}
private Container getContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 42dcd6d..69b0813 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -1311,7 +1311,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(2*GB, node_0.getUsedResource().getMemory());
+ assertEquals(2*GB, node_0.getAllocatedResource().getMemory());
assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
@@ -1328,7 +1328,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(1*GB, node_0.getUsedResource().getMemory());
+ assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
@@ -1345,7 +1345,7 @@ public class TestLeafQueue {
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(4*GB, node_0.getUsedResource().getMemory());
+ assertEquals(4*GB, node_0.getAllocatedResource().getMemory());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(4*GB, a.getMetrics().getAllocatedMB());
}
@@ -1434,7 +1434,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(2*GB, node_0.getUsedResource().getMemory());
+ assertEquals(2*GB, node_0.getAllocatedResource().getMemory());
// Now free 1 container from app_0 i.e. 1G, and re-reserve it
RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
@@ -1449,7 +1449,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(1*GB, node_0.getUsedResource().getMemory());
+ assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(1, app_1.getReReservations(priority));
// Re-reserve
@@ -1459,7 +1459,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(1*GB, node_0.getUsedResource().getMemory());
+ assertEquals(1*GB, node_0.getAllocatedResource().getMemory());
assertEquals(2, app_1.getReReservations(priority));
// Try to schedule on node_1 now, should *move* the reservation
@@ -1469,7 +1469,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(4*GB, node_1.getUsedResource().getMemory());
+ assertEquals(4*GB, node_1.getAllocatedResource().getMemory());
// Doesn't change yet... only when reservation is cancelled or a different
// container is reserved
assertEquals(2, app_1.getReReservations(priority));
@@ -1487,7 +1487,7 @@ public class TestLeafQueue {
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
- assertEquals(0*GB, node_0.getUsedResource().getMemory());
+ assertEquals(0*GB, node_0.getAllocatedResource().getMemory());
}
private void verifyContainerAllocated(CSAssignment assignment, NodeType nodeType) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 4a815f5..f73baa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -158,9 +158,9 @@ public class TestParentQueue {
any(ResourceLimits.class), any(SchedulingMode.class));
// Mock the node's resource availability
- Resource available = node.getAvailableResource();
+ Resource available = node.getUnallocatedResource();
doReturn(Resources.subtractFrom(available, allocatedResource)).
- when(node).getAvailableResource();
+ when(node).getUnallocatedResource();
}
return new CSAssignment(allocatedResource, type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 9b920d0..9047138 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -263,9 +263,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -275,9 +275,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -289,9 +289,9 @@ public class TestReservations {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@@ -305,9 +305,9 @@ public class TestReservations {
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// assign reducer to node 2
@@ -321,9 +321,9 @@ public class TestReservations {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
// node_1 heartbeat and unreserves from node_0 in order to allocate
@@ -337,9 +337,9 @@ public class TestReservations {
assertEquals(6 * GB, a.getMetrics().getAvailableMB());
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(8 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(8 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(0, app_0.getTotalRequiredResources(priorityReduce));
}
@@ -425,9 +425,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
@@ -437,9 +437,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
assertEquals(20 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(2 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(2 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Add a few requests to each app
app_0.updateResourceRequests(Collections.singletonList(TestUtils
@@ -458,9 +458,9 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getReservedMB());
assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
assertEquals(12 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(2 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(2 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// next assignment is beyond user limit for user_0 but it should assign to
// app_1 for user_1
@@ -472,9 +472,9 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getReservedMB());
assertEquals(6 * GB, a.getMetrics().getAllocatedMB());
assertEquals(10 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(4 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(4 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
}
@Test
@@ -559,9 +559,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -571,9 +571,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -585,9 +585,9 @@ public class TestReservations {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@@ -601,9 +601,9 @@ public class TestReservations {
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// assign reducer to node 2
@@ -617,9 +617,9 @@ public class TestReservations {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
// node_1 heartbeat and won't unreserve from node_0, potentially stuck
@@ -634,9 +634,9 @@ public class TestReservations {
assertEquals(6 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
}
@@ -718,8 +718,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -729,8 +729,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -742,8 +742,8 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(8 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// try to assign reducer (5G on node 0 and should reserve)
@@ -757,8 +757,8 @@ public class TestReservations {
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(5 * GB, node_0.getReservedContainer().getReservedResource()
.getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// could allocate but told need to unreserve first
@@ -771,8 +771,8 @@ public class TestReservations {
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(8 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(8 * GB, node_1.getAllocatedResource().getMemory());
assertEquals(1, app_0.getTotalRequiredResources(priorityReduce));
}
@@ -981,8 +981,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -992,8 +992,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -1004,8 +1004,8 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// now add in reservations and make sure it continues if config set
// allocate to queue so that the potential new capacity is greater then
@@ -1018,8 +1018,8 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
ResourceLimits limits =
new ResourceLimits(Resources.createResource(13 * GB));
@@ -1155,8 +1155,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(14 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -1166,8 +1166,8 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -1178,8 +1178,8 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(8 * GB, a.getMetrics().getAvailableMB());
assertEquals(null, node_0.getReservedContainer());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// now add in reservations and make sure it continues if config set
// allocate to queue so that the potential new capacity is greater then
@@ -1194,8 +1194,8 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
// not over the limit
Resource limit = Resources.createResource(14 * GB, 0);
@@ -1307,9 +1307,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
assertEquals(22 * GB, a.getMetrics().getAvailableMB());
- assertEquals(2 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map - simulating reduce
a.assignContainers(clusterResource, node_0,
@@ -1319,9 +1319,9 @@ public class TestReservations {
assertEquals(0 * GB, a.getMetrics().getReservedMB());
assertEquals(5 * GB, a.getMetrics().getAllocatedMB());
assertEquals(19 * GB, a.getMetrics().getAvailableMB());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(0 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// Only 1 map to other node - simulating reduce
a.assignContainers(clusterResource, node_1,
@@ -1332,9 +1332,9 @@ public class TestReservations {
assertEquals(8 * GB, a.getMetrics().getAllocatedMB());
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
assertEquals(16 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// try to assign reducer (5G on node 0), but tell it's resource limits <
// used (8G) + required (5G). It will not reserved since it has to unreserve
@@ -1349,9 +1349,9 @@ public class TestReservations {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
// app_0's headroom = limit (10G) - used (8G) = 2G
assertEquals(2 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// try to assign reducer (5G on node 0), but tell it's resource limits <
// used (8G) + required (5G). It will not reserved since it has to unreserve
@@ -1365,9 +1365,9 @@ public class TestReservations {
assertEquals(16 * GB, a.getMetrics().getAvailableMB());
// app_0's headroom = limit (10G) - used (8G) = 2G
assertEquals(2 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(0 * GB, node_2.getAllocatedResource().getMemory());
// let it assign 5G to node_2
a.assignContainers(clusterResource, node_2,
@@ -1378,9 +1378,9 @@ public class TestReservations {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(11 * GB, a.getMetrics().getAvailableMB());
assertEquals(11 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
// reserve 8G node_0
a.assignContainers(clusterResource, node_0,
@@ -1391,9 +1391,9 @@ public class TestReservations {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
// try to assign (8G on node 2). No room to allocate,
// continued to try due to having reservation above,
@@ -1406,8 +1406,8 @@ public class TestReservations {
assertEquals(13 * GB, a.getMetrics().getAllocatedMB());
assertEquals(3 * GB, a.getMetrics().getAvailableMB());
assertEquals(3 * GB, app_0.getHeadroom().getMemory());
- assertEquals(5 * GB, node_0.getUsedResource().getMemory());
- assertEquals(3 * GB, node_1.getUsedResource().getMemory());
- assertEquals(5 * GB, node_2.getUsedResource().getMemory());
+ assertEquals(5 * GB, node_0.getAllocatedResource().getMemory());
+ assertEquals(3 * GB, node_1.getAllocatedResource().getMemory());
+ assertEquals(5 * GB, node_2.getAllocatedResource().getMemory());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 1786069..2694957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -195,7 +195,7 @@ public class TestUtils {
when(rmNode.getRackName()).thenReturn(rack);
FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
- LOG.info("node = " + host + " avail=" + node.getAvailableResource());
+ LOG.info("node = " + host + " avail=" + node.getUnallocatedResource());
when(node.getNodeID()).thenReturn(nodeId);
return node;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index d29d92c..e733b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -318,7 +318,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
.when(spyApp).isWaitingForAMContainer();
assertTrue(spyApp.isBlacklisted(n1.getNodeName()));
assertFalse(spyApp.isBlacklisted(n2.getNodeName()));
- assertEquals(n2.getAvailableResource(), spyApp.getHeadroom());
+ assertEquals(n2.getUnallocatedResource(), spyApp.getHeadroom());
blacklistAdditions.clear();
blacklistAdditions.add(n2.getNodeName());
@@ -326,7 +326,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
app.updateBlacklist(blacklistAdditions, blacklistRemovals);
assertFalse(spyApp.isBlacklisted(n1.getNodeName()));
assertTrue(spyApp.isBlacklisted(n2.getNodeName()));
- assertEquals(n1.getAvailableResource(), spyApp.getHeadroom());
+ assertEquals(n1.getUnallocatedResource(), spyApp.getHeadroom());
blacklistAdditions.clear();
blacklistRemovals.clear();
[37/50] [abbrv] hadoop git commit: HDFS-9887. WebHdfs socket timeouts
should be configurable. Contributed by Austin Donnelly and Chris Douglas.
Posted by ar...@apache.org.
HDFS-9887. WebHdfs socket timeouts should be configurable. Contributed by Austin Donnelly and Chris Douglas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5abf0512
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5abf0512
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5abf0512
Branch: refs/heads/HDFS-1312
Commit: 5abf051249d485313dfffc6aeff6f81c0da1f623
Parents: 27e0681
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Mar 2 09:23:17 2016 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Mar 2 09:23:17 2016 -0800
----------------------------------------------------------------------
.../hdfs/client/HdfsClientConfigKeys.java | 5 +++
.../hadoop/hdfs/web/URLConnectionFactory.java | 30 ++++++++++++----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 ++-
.../src/main/resources/hdfs-default.xml | 27 ++++++++++++++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 2 ++
.../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 37 +++++++++++++++++++-
6 files changed, 98 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index ce0f3e4..f036c56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -38,6 +38,11 @@ public interface HdfsClientConfigKeys {
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
+ String DFS_WEBHDFS_SOCKET_CONNECT_TIMEOUT_KEY =
+ "dfs.webhdfs.socket.connect-timeout";
+ String DFS_WEBHDFS_SOCKET_READ_TIMEOUT_KEY =
+ "dfs.webhdfs.socket.read-timeout";
+
String DFS_WEBHDFS_OAUTH_ENABLED_KEY = "dfs.webhdfs.oauth2.enabled";
boolean DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
index 8810ac4..975f72e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
@@ -23,6 +23,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.security.GeneralSecurityException;
+import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
@@ -31,6 +32,7 @@ import javax.net.ssl.SSLSocketFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.web.oauth2.OAuth2ConnectionConfigurator;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
@@ -62,7 +64,9 @@ public class URLConnectionFactory {
@Override
public HttpURLConnection configure(HttpURLConnection conn)
throws IOException {
- URLConnectionFactory.setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
+ URLConnectionFactory.setTimeouts(conn,
+ DEFAULT_SOCKET_TIMEOUT,
+ DEFAULT_SOCKET_TIMEOUT);
return conn;
}
};
@@ -128,17 +132,29 @@ public class URLConnectionFactory {
* Create a new ConnectionConfigurator for SSL connections
*/
private static ConnectionConfigurator newSslConnConfigurator(
- final int timeout, Configuration conf)
+ final int defaultTimeout, Configuration conf)
throws IOException, GeneralSecurityException {
final SSLFactory factory;
final SSLSocketFactory sf;
final HostnameVerifier hv;
+ final int connectTimeout;
+ final int readTimeout;
factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
factory.init();
sf = factory.createSSLSocketFactory();
hv = factory.getHostnameVerifier();
+ connectTimeout = (int) conf.getTimeDuration(
+ HdfsClientConfigKeys.DFS_WEBHDFS_SOCKET_CONNECT_TIMEOUT_KEY,
+ defaultTimeout,
+ TimeUnit.MILLISECONDS);
+
+ readTimeout = (int) conf.getTimeDuration(
+ HdfsClientConfigKeys.DFS_WEBHDFS_SOCKET_READ_TIMEOUT_KEY,
+ defaultTimeout,
+ TimeUnit.MILLISECONDS);
+
return new ConnectionConfigurator() {
@Override
public HttpURLConnection configure(HttpURLConnection conn)
@@ -148,7 +164,7 @@ public class URLConnectionFactory {
c.setSSLSocketFactory(sf);
c.setHostnameVerifier(hv);
}
- URLConnectionFactory.setTimeouts(conn, timeout);
+ URLConnectionFactory.setTimeouts(conn, connectTimeout, readTimeout);
return conn;
}
};
@@ -208,8 +224,10 @@ public class URLConnectionFactory {
* @param socketTimeout
* the connection and read timeout of the connection.
*/
- private static void setTimeouts(URLConnection connection, int socketTimeout) {
- connection.setConnectTimeout(socketTimeout);
- connection.setReadTimeout(socketTimeout);
+ private static void setTimeouts(URLConnection connection,
+ int connectTimeout,
+ int readTimeout) {
+ connection.setConnectTimeout(connectTimeout);
+ connection.setReadTimeout(readTimeout);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8be05bf..9562949 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1993,9 +1993,12 @@ Release 2.8.0 - UNRELEASED
HDFS-9843. Document distcp options required for copying between encrypted
locations. (Xiaoyu Yao via cnauroth)
- HDFS-9831.Document webhdfs retry configuration keys introduced by
+ HDFS-9831. Document webhdfs retry configuration keys introduced by
HDFS-5219/HDFS-5122. (Xiaobing Zhou via xyao)
+ HDFS-9887. WebHdfs socket timeouts should be configurable.
+ (Austin Donnelly and Chris Douglas via xyao)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b4fb2e0..ea25a91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2318,6 +2318,33 @@
</property>
<property>
+ <name>dfs.webhdfs.socket.connect-timeout</name>
+ <value>60s</value>
+ <description>
+ Socket timeout for connecting to WebHDFS servers. This prevents a
+ WebHDFS client from hanging if the server hostname is
+ misconfigured, or the server does not response before the timeout
+ expires. Value is followed by a unit specifier: ns, us, ms, s, m,
+ h, d for nanoseconds, microseconds, milliseconds, seconds,
+ minutes, hours, days respectively. Values should provide units,
+ but milliseconds are assumed.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.socket.read-timeout</name>
+ <value>60s</value>
+ <description>
+ Socket timeout for reading data from WebHDFS servers. This
+ prevents a WebHDFS client from hanging if the server stops sending
+ data. Value is followed by a unit specifier: ns, us, ms, s, m, h,
+ d for nanoseconds, microseconds, milliseconds, seconds, minutes,
+ hours, days respectively. Values should provide units,
+ but milliseconds are assumed.
+ </description>
+</property>
+
+<property>
<name>dfs.client.context</name>
<value>default</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 79e79b2..c719482 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -199,6 +199,8 @@ Below are the HDFS configuration options for WebHDFS.
|:---- |:---- |
| `dfs.web.authentication.kerberos.principal` | The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint. The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos HTTP SPNEGO specification. A value of "\*" will use all HTTP principals found in the keytab. |
| `dfs.web.authentication.kerberos.keytab ` | The Kerberos keytab file with the credentials for the HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint. |
+| `dfs.webhdfs.socket.connect-timeout` | How long to wait for a connection to be established before failing. Specified as a time duration, ie numerical value followed by a units symbol, eg 2m for two minutes. Defaults to 60s. |
+| `dfs.webhdfs.socket.read-timeout` | How long to wait for data to arrive before failing. Defaults to 60s. |
Authentication
--------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abf0512/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
index 3a87d42..664e32d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
@@ -33,18 +33,25 @@ import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.junit.After;
import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
import org.junit.Test;
/**
@@ -54,6 +61,7 @@ import org.junit.Test;
* bogus server on the namenode HTTP port, which is rigged to not accept new
* connections or to accept connections but not send responses.
*/
+@RunWith(Parameterized.class)
public class TestWebHdfsTimeouts {
private static final Log LOG = LogFactory.getLog(TestWebHdfsTimeouts.class);
@@ -77,14 +85,41 @@ public class TestWebHdfsTimeouts {
}
});
+ public enum TimeoutSource { ConnectionFactory, Configuration };
+
+ /**
+ * Run all tests twice: once with the timeouts set by the
+ * connection factory, and again with the timeouts set by
+ * configuration options.
+ */
+ @Parameters(name = "timeoutSource={0}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { TimeoutSource.ConnectionFactory },
+ { TimeoutSource.Configuration }
+ });
+ }
+
+ @Parameter
+ public TimeoutSource timeoutSource;
+
@Before
public void setUp() throws Exception {
Configuration conf = WebHdfsTestUtil.createConf();
serverSocket = new ServerSocket(0, CONNECTION_BACKLOG);
nnHttpAddress = new InetSocketAddress("localhost", serverSocket.getLocalPort());
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "localhost:" + serverSocket.getLocalPort());
+ if (timeoutSource == TimeoutSource.Configuration) {
+ String v = Integer.toString(SHORT_SOCKET_TIMEOUT) + "ms";
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_SOCKET_CONNECT_TIMEOUT_KEY, v);
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_SOCKET_READ_TIMEOUT_KEY, v);
+ }
+
fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
- fs.connectionFactory = connectionFactory;
+ if (timeoutSource == TimeoutSource.ConnectionFactory) {
+ fs.connectionFactory = connectionFactory;
+ }
+
clients = new ArrayList<SocketChannel>();
serverThread = null;
}
[06/50] [abbrv] hadoop git commit: YARN-4718. Rename variables in
SchedulerNode to reduce ambiguity post YARN-1011. (Inigo Goiri via kasha)
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index a15e8d1..8d7c22e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4433,7 +4433,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getUsedResource();
+ .getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 0);
Assert.assertEquals(usedResource.getVirtualCores(), 0);
// Check total resource of scheduler node is also changed to 0 GB 0 core
@@ -4445,7 +4445,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
+ .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9692770/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 86a017e..9bfc283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -348,7 +348,7 @@ public class TestFifoScheduler {
assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
.getMemory(), 1024);
assertEquals(schedulerNodes.get(node0.getNodeID()).
- getAvailableResource().getMemory(), 1024);
+ getUnallocatedResource().getMemory(), 1024);
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
@@ -1247,7 +1247,7 @@ public class TestFifoScheduler {
// Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
Resource usedResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getUsedResource();
+ .getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
Assert.assertEquals(usedResource.getMemory(), 1 * GB);
Assert.assertEquals(usedResource.getVirtualCores(), 1);
// Check total resource of scheduler node is also changed to 1 GB 1 core
@@ -1259,7 +1259,7 @@ public class TestFifoScheduler {
// Check the available resource is 0/0
Resource availableResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getAvailableResource();
+ .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
Assert.assertEquals(availableResource.getMemory(), 0);
Assert.assertEquals(availableResource.getVirtualCores(), 0);
}