You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2014/07/25 22:33:22 UTC
svn commit: r1613514 [4/6] - in
/hadoop/common/branches/YARN-1051/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/
hadoop-hdfs-nfs/src/test/java/org/apac...
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Fri Jul 25 20:33:09 2014
@@ -28,6 +28,7 @@ import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import javax.servlet.ServletContext;
@@ -84,6 +85,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.DestinationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
+import org.apache.hadoop.hdfs.web.resources.ExcludeDatanodesParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.GroupParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
@@ -111,13 +113,16 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.XAttrSetFlagParam;
import org.apache.hadoop.hdfs.web.resources.XAttrValueParam;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
+import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.StringUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
@@ -186,16 +191,30 @@ public class NamenodeWebHdfsMethods {
throws IOException {
final NamenodeProtocols np = namenode.getRpcServer();
if (np == null) {
- throw new IOException("Namenode is in startup mode");
+ throw new RetriableException("Namenode is in startup mode");
}
return np;
}
-
+
@VisibleForTesting
static DatanodeInfo chooseDatanode(final NameNode namenode,
final String path, final HttpOpParam.Op op, final long openOffset,
- final long blocksize) throws IOException {
+ final long blocksize, final String excludeDatanodes) throws IOException {
final BlockManager bm = namenode.getNamesystem().getBlockManager();
+
+ HashSet<Node> excludes = new HashSet<Node>();
+ if (excludeDatanodes != null) {
+ for (String host : StringUtils
+ .getTrimmedStringCollection(excludeDatanodes)) {
+ int idx = host.indexOf(":");
+ if (idx != -1) {
+ excludes.add(bm.getDatanodeManager().getDatanodeByXferAddr(
+ host.substring(0, idx), Integer.parseInt(host.substring(idx + 1))));
+ } else {
+ excludes.add(bm.getDatanodeManager().getDatanodeByHost(host));
+ }
+ }
+ }
if (op == PutOpParam.Op.CREATE) {
//choose a datanode near to client
@@ -204,7 +223,7 @@ public class NamenodeWebHdfsMethods {
if (clientNode != null) {
final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy()
.chooseTarget(path, 1, clientNode,
- new ArrayList<DatanodeStorageInfo>(), false, null, blocksize,
+ new ArrayList<DatanodeStorageInfo>(), false, excludes, blocksize,
// TODO: get storage type from the file
StorageType.DEFAULT);
if (storages.length > 0) {
@@ -233,7 +252,7 @@ public class NamenodeWebHdfsMethods {
final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
final int count = locations.locatedBlockCount();
if (count > 0) {
- return bestNode(locations.get(0).getLocations());
+ return bestNode(locations.get(0).getLocations(), excludes);
}
}
}
@@ -247,11 +266,14 @@ public class NamenodeWebHdfsMethods {
* sorted based on availability and network distances, thus it is sufficient
* to return the first element of the node here.
*/
- private static DatanodeInfo bestNode(DatanodeInfo[] nodes) throws IOException {
- if (nodes.length == 0 || nodes[0].isDecommissioned()) {
- throw new IOException("No active nodes contain this block");
+ private static DatanodeInfo bestNode(DatanodeInfo[] nodes,
+ HashSet<Node> excludes) throws IOException {
+ for (DatanodeInfo dn: nodes) {
+ if (false == dn.isDecommissioned() && false == excludes.contains(dn)) {
+ return dn;
+ }
}
- return nodes[0];
+ throw new IOException("No active nodes contain this block");
}
private Token<? extends TokenIdentifier> generateDelegationToken(
@@ -270,11 +292,12 @@ public class NamenodeWebHdfsMethods {
final UserGroupInformation ugi, final DelegationParam delegation,
final UserParam username, final DoAsParam doAsUser,
final String path, final HttpOpParam.Op op, final long openOffset,
- final long blocksize,
+ final long blocksize, final String excludeDatanodes,
final Param<?, ?>... parameters) throws URISyntaxException, IOException {
final DatanodeInfo dn;
try {
- dn = chooseDatanode(namenode, path, op, openOffset, blocksize);
+ dn = chooseDatanode(namenode, path, op, openOffset, blocksize,
+ excludeDatanodes);
} catch (InvalidTopologyException ite) {
throw new IOException("Failed to find datanode, suggest to check cluster health.", ite);
}
@@ -361,13 +384,15 @@ public class NamenodeWebHdfsMethods {
@QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT)
final SnapshotNameParam snapshotName,
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
- final OldSnapshotNameParam oldSnapshotName
- )throws IOException, InterruptedException {
+ final OldSnapshotNameParam oldSnapshotName,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
+ ) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
owner, group, permission, overwrite, bufferSize, replication,
blockSize, modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
}
/** Handle HTTP PUT request. */
@@ -423,14 +448,16 @@ public class NamenodeWebHdfsMethods {
@QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT)
final SnapshotNameParam snapshotName,
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
- final OldSnapshotNameParam oldSnapshotName
+ final OldSnapshotNameParam oldSnapshotName,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
group, permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName);
+ oldSnapshotName, excludeDatanodes);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -441,7 +468,7 @@ public class NamenodeWebHdfsMethods {
permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
} finally {
reset();
}
@@ -474,7 +501,8 @@ public class NamenodeWebHdfsMethods {
final XAttrValueParam xattrValue,
final XAttrSetFlagParam xattrSetFlag,
final SnapshotNameParam snapshotName,
- final OldSnapshotNameParam oldSnapshotName
+ final OldSnapshotNameParam oldSnapshotName,
+ final ExcludeDatanodesParam exclDatanodes
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
@@ -484,9 +512,10 @@ public class NamenodeWebHdfsMethods {
switch(op.getValue()) {
case CREATE:
{
- final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
- fullpath, op.getValue(), -1L, blockSize.getValue(conf),
- permission, overwrite, bufferSize, replication, blockSize);
+ final URI uri = redirectURI(namenode, ugi, delegation, username,
+ doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
+ exclDatanodes.getValue(), permission, overwrite, bufferSize,
+ replication, blockSize);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case MKDIRS:
@@ -619,9 +648,12 @@ public class NamenodeWebHdfsMethods {
@QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
final ConcatSourcesParam concatSrcs,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
- final BufferSizeParam bufferSize
+ final BufferSizeParam bufferSize,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, InterruptedException {
- return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs, bufferSize);
+ return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs,
+ bufferSize, excludeDatanodes);
}
/** Handle HTTP POST request. */
@@ -643,17 +675,21 @@ public class NamenodeWebHdfsMethods {
@QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
final ConcatSourcesParam concatSrcs,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
- final BufferSizeParam bufferSize
+ final BufferSizeParam bufferSize,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, InterruptedException {
- init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize);
+ init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize,
+ excludeDatanodes);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
public Response run() throws IOException, URISyntaxException {
try {
return post(ugi, delegation, username, doAsUser,
- path.getAbsolutePath(), op, concatSrcs, bufferSize);
+ path.getAbsolutePath(), op, concatSrcs, bufferSize,
+ excludeDatanodes);
} finally {
reset();
}
@@ -669,15 +705,17 @@ public class NamenodeWebHdfsMethods {
final String fullpath,
final PostOpParam op,
final ConcatSourcesParam concatSrcs,
- final BufferSizeParam bufferSize
+ final BufferSizeParam bufferSize,
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, URISyntaxException {
final NameNode namenode = (NameNode)context.getAttribute("name.node");
switch(op.getValue()) {
case APPEND:
{
- final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
- fullpath, op.getValue(), -1L, -1L, bufferSize);
+ final URI uri = redirectURI(namenode, ugi, delegation, username,
+ doAsUser, fullpath, op.getValue(), -1L, -1L,
+ excludeDatanodes.getValue(), bufferSize);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case CONCAT:
@@ -715,10 +753,12 @@ public class NamenodeWebHdfsMethods {
@QueryParam(XAttrNameParam.NAME) @DefaultValue(XAttrNameParam.DEFAULT)
final List<XAttrNameParam> xattrNames,
@QueryParam(XAttrEncodingParam.NAME) @DefaultValue(XAttrEncodingParam.DEFAULT)
- final XAttrEncodingParam xattrEncoding
+ final XAttrEncodingParam xattrEncoding,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, InterruptedException {
return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
- renewer, bufferSize, xattrNames, xattrEncoding);
+ renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes);
}
/** Handle HTTP GET request. */
@@ -747,11 +787,13 @@ public class NamenodeWebHdfsMethods {
@QueryParam(XAttrNameParam.NAME) @DefaultValue(XAttrNameParam.DEFAULT)
final List<XAttrNameParam> xattrNames,
@QueryParam(XAttrEncodingParam.NAME) @DefaultValue(XAttrEncodingParam.DEFAULT)
- final XAttrEncodingParam xattrEncoding
+ final XAttrEncodingParam xattrEncoding,
+ @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, offset, length,
- renewer, bufferSize, xattrEncoding);
+ renewer, bufferSize, xattrEncoding, excludeDatanodes);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -759,7 +801,7 @@ public class NamenodeWebHdfsMethods {
try {
return get(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, offset, length, renewer, bufferSize,
- xattrNames, xattrEncoding);
+ xattrNames, xattrEncoding, excludeDatanodes);
} finally {
reset();
}
@@ -779,7 +821,8 @@ public class NamenodeWebHdfsMethods {
final RenewerParam renewer,
final BufferSizeParam bufferSize,
final List<XAttrNameParam> xattrNames,
- final XAttrEncodingParam xattrEncoding
+ final XAttrEncodingParam xattrEncoding,
+ final ExcludeDatanodesParam excludeDatanodes
) throws IOException, URISyntaxException {
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final NamenodeProtocols np = getRPCServer(namenode);
@@ -787,8 +830,9 @@ public class NamenodeWebHdfsMethods {
switch(op.getValue()) {
case OPEN:
{
- final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
- fullpath, op.getValue(), offset.getValue(), -1L, offset, length, bufferSize);
+ final URI uri = redirectURI(namenode, ugi, delegation, username,
+ doAsUser, fullpath, op.getValue(), offset.getValue(), -1L,
+ excludeDatanodes.getValue(), offset, length, bufferSize);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case GET_BLOCK_LOCATIONS:
@@ -824,7 +868,7 @@ public class NamenodeWebHdfsMethods {
case GETFILECHECKSUM:
{
final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
- fullpath, op.getValue(), -1L, -1L);
+ fullpath, op.getValue(), -1L, -1L, null);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case GETDELEGATIONTOKEN:
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java Fri Jul 25 20:33:09 2014
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
@@ -50,6 +51,7 @@ public class BlockCommand extends Datano
final String poolId;
final Block[] blocks;
final DatanodeInfo[][] targets;
+ final StorageType[][] targetStorageTypes;
final String[][] targetStorageIDs;
/**
@@ -62,17 +64,20 @@ public class BlockCommand extends Datano
this.poolId = poolId;
blocks = new Block[blocktargetlist.size()];
targets = new DatanodeInfo[blocks.length][];
+ targetStorageTypes = new StorageType[blocks.length][];
targetStorageIDs = new String[blocks.length][];
for(int i = 0; i < blocks.length; i++) {
BlockTargetPair p = blocktargetlist.get(i);
blocks[i] = p.block;
targets[i] = DatanodeStorageInfo.toDatanodeInfos(p.targets);
+ targetStorageTypes[i] = DatanodeStorageInfo.toStorageTypes(p.targets);
targetStorageIDs[i] = DatanodeStorageInfo.toStorageIDs(p.targets);
}
}
private static final DatanodeInfo[][] EMPTY_TARGET_DATANODES = {};
+ private static final StorageType[][] EMPTY_TARGET_STORAGE_TYPES = {};
private static final String[][] EMPTY_TARGET_STORAGEIDS = {};
/**
@@ -81,7 +86,7 @@ public class BlockCommand extends Datano
*/
public BlockCommand(int action, String poolId, Block blocks[]) {
this(action, poolId, blocks, EMPTY_TARGET_DATANODES,
- EMPTY_TARGET_STORAGEIDS);
+ EMPTY_TARGET_STORAGE_TYPES, EMPTY_TARGET_STORAGEIDS);
}
/**
@@ -89,11 +94,13 @@ public class BlockCommand extends Datano
* @param blocks blocks related to the action
*/
public BlockCommand(int action, String poolId, Block[] blocks,
- DatanodeInfo[][] targets, String[][] targetStorageIDs) {
+ DatanodeInfo[][] targets, StorageType[][] targetStorageTypes,
+ String[][] targetStorageIDs) {
super(action);
this.poolId = poolId;
this.blocks = blocks;
this.targets = targets;
+ this.targetStorageTypes = targetStorageTypes;
this.targetStorageIDs = targetStorageIDs;
}
@@ -109,6 +116,10 @@ public class BlockCommand extends Datano
return targets;
}
+ public StorageType[][] getTargetStorageTypes() {
+ return targetStorageTypes;
+ }
+
public String[][] getTargetStorageIDs() {
return targetStorageIDs;
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java Fri Jul 25 20:33:09 2014
@@ -122,6 +122,11 @@ public class DFSZKFailoverController ext
"HA is not enabled for this namenode.");
}
String nnId = HAUtil.getNameNodeId(localNNConf, nsId);
+ if (nnId == null) {
+ String msg = "Could not get the namenode ID of this node. " +
+ "You may run zkfc on the node other than namenode.";
+ throw new HadoopIllegalArgumentException(msg);
+ }
NameNode.initializeGenericKeys(localNNConf, nsId, nnId);
DFSUtil.setGenericConf(localNNConf, nsId, nnId, ZKFC_CONF_KEYS);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java Fri Jul 25 20:33:09 2014
@@ -77,7 +77,7 @@ public class DFSck extends Configured im
private static final String USAGE = "Usage: DFSck <path> "
+ "[-list-corruptfileblocks | "
+ "[-move | -delete | -openforwrite] "
- + "[-files [-blocks [-locations | -racks]]]]\n"
+ + "[-files [-blocks [-locations | -racks]]]] [-showprogress]\n"
+ "\t<path>\tstart checking from this path\n"
+ "\t-move\tmove corrupted files to /lost+found\n"
+ "\t-delete\tdelete corrupted files\n"
@@ -90,7 +90,8 @@ public class DFSck extends Configured im
+ "blocks and files they belong to\n"
+ "\t-blocks\tprint out block report\n"
+ "\t-locations\tprint out locations for every block\n"
- + "\t-racks\tprint out network topology for data-node locations\n\n"
+ + "\t-racks\tprint out network topology for data-node locations\n"
+ + "\t-showprogress\tshow progress in output. Default is OFF (no progress)\n\n"
+ "Please Note:\n"
+ "\t1. By default fsck ignores files opened for write, "
+ "use -openforwrite to report such files. They are usually "
@@ -270,6 +271,7 @@ public class DFSck extends Configured im
else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); }
else if (args[idx].equals("-locations")) { url.append("&locations=1"); }
else if (args[idx].equals("-racks")) { url.append("&racks=1"); }
+ else if (args[idx].equals("-showprogress")) { url.append("&showprogress=1"); }
else if (args[idx].equals("-list-corruptfileblocks")) {
url.append("&listcorruptfileblocks=1");
doListCorruptFileBlocks = true;
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Fri Jul 25 20:33:09 2014
@@ -158,7 +158,7 @@ public class WebHdfsFileSystem extends F
// getCanonicalUri() in order to handle the case where no port is
// specified in the URI
this.tokenServiceName = isLogicalUri ?
- HAUtil.buildTokenServiceForLogicalUri(uri)
+ HAUtil.buildTokenServiceForLogicalUri(uri, getScheme())
: SecurityUtil.buildTokenService(getCanonicalUri());
if (!isHA) {
@@ -448,6 +448,7 @@ public class WebHdfsFileSystem extends F
protected final HttpOpParam.Op op;
private final boolean redirected;
+ protected ExcludeDatanodesParam excludeDatanodes = new ExcludeDatanodesParam("");
private boolean checkRetry;
@@ -499,6 +500,10 @@ public class WebHdfsFileSystem extends F
* a DN such as open and checksum
*/
private HttpURLConnection connect(URL url) throws IOException {
+ //redirect hostname and port
+ String redirectHost = null;
+
+
// resolve redirects for a DN operation unless already resolved
if (op.getRedirect() && !redirected) {
final HttpOpParam.Op redirectOp =
@@ -511,11 +516,24 @@ public class WebHdfsFileSystem extends F
try {
validateResponse(redirectOp, conn, false);
url = new URL(conn.getHeaderField("Location"));
+ redirectHost = url.getHost() + ":" + url.getPort();
} finally {
conn.disconnect();
}
}
- return connect(op, url);
+ try {
+ return connect(op, url);
+ } catch (IOException ioe) {
+ if (redirectHost != null) {
+ if (excludeDatanodes.getValue() != null) {
+ excludeDatanodes = new ExcludeDatanodesParam(redirectHost + ","
+ + excludeDatanodes.getValue());
+ } else {
+ excludeDatanodes = new ExcludeDatanodesParam(redirectHost);
+ }
+ }
+ throw ioe;
+ }
}
private HttpURLConnection connect(final HttpOpParam.Op op, final URL url)
@@ -652,7 +670,14 @@ public class WebHdfsFileSystem extends F
@Override
protected URL getUrl() throws IOException {
- return toUrl(op, fspath, parameters);
+ if (excludeDatanodes.getValue() != null) {
+ Param<?, ?>[] tmpParam = new Param<?, ?>[parameters.length + 1];
+ System.arraycopy(parameters, 0, tmpParam, 0, parameters.length);
+ tmpParam[parameters.length] = excludeDatanodes;
+ return toUrl(op, fspath, tmpParam);
+ } else {
+ return toUrl(op, fspath, parameters);
+ }
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java Fri Jul 25 20:33:09 2014
@@ -25,8 +25,8 @@ public class XAttrNameParam extends Stri
/** Default parameter value. **/
public static final String DEFAULT = "";
- private static Domain DOMAIN = new Domain(NAME,
- Pattern.compile("^(user\\.|trusted\\.|system\\.|security\\.).+"));
+ private static Domain DOMAIN = new Domain(NAME,
+ Pattern.compile(".*"));
public XAttrNameParam(final String str) {
super(DOMAIN, str == null || str.equals(DEFAULT) ? null : str);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto Fri Jul 25 20:33:09 2014
@@ -113,6 +113,7 @@ message BlockCommandProto {
repeated BlockProto blocks = 3;
repeated DatanodeInfosProto targets = 4;
repeated StorageUuidsProto targetStorageUuids = 5;
+ repeated StorageTypesProto targetStorageTypes = 6;
}
/**
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto Fri Jul 25 20:33:09 2014
@@ -107,17 +107,21 @@ message OpWriteBlockProto {
*/
required ChecksumProto requestedChecksum = 9;
optional CachingStrategyProto cachingStrategy = 10;
+ optional StorageTypeProto storageType = 11 [default = DISK];
+ repeated StorageTypeProto targetStorageTypes = 12;
}
message OpTransferBlockProto {
required ClientOperationHeaderProto header = 1;
repeated DatanodeInfoProto targets = 2;
+ repeated StorageTypeProto targetStorageTypes = 3;
}
message OpReplaceBlockProto {
required BaseHeaderProto header = 1;
required string delHint = 2;
required DatanodeInfoProto source = 3;
+ optional StorageTypeProto storageType = 4 [default = DISK];
}
message OpCopyBlockProto {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Fri Jul 25 20:33:09 2014
@@ -137,6 +137,13 @@ enum StorageTypeProto {
}
/**
+ * A list of storage types.
+ */
+message StorageTypesProto {
+ repeated StorageTypeProto storageTypes = 1;
+}
+
+/**
* A list of storage IDs.
*/
message StorageUuidsProto {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Fri Jul 25 20:33:09 2014
@@ -1452,6 +1452,37 @@
</property>
<property>
+ <name>dfs.data.transfer.protection</name>
+ <value></value>
+ <description>
+ A comma-separated list of SASL protection values used for secured
+ connections to the DataNode when reading or writing block data. Possible
+ values are authentication, integrity and privacy. authentication means
+ authentication only and no integrity or privacy; integrity implies
+ authentication and integrity are enabled; and privacy implies all of
+ authentication, integrity and privacy are enabled. If
+ dfs.encrypt.data.transfer is set to true, then it supersedes the setting for
+ dfs.data.transfer.protection and enforces that all connections must use a
+ specialized encrypted SASL handshake. This property is ignored for
+ connections to a DataNode listening on a privileged port. In this case, it
+ is assumed that the use of a privileged port establishes sufficient trust.
+ </description>
+</property>
+
+<property>
+ <name>dfs.data.transfer.saslproperties.resolver.class</name>
+ <value></value>
+ <description>
+ SaslPropertiesResolver used to resolve the QOP used for a connection to the
+ DataNode when reading or writing block data. If not specified, the full set
+ of values specified in dfs.data.transfer.protection is used while
+ determining the QOP used for the connection. If a class is specified, then
+ the QOP values returned by the class will be used while determining the QOP
+ used for the connection.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>false</value>
<description>
@@ -2008,4 +2039,17 @@
</description>
</property>
+<property>
+ <name>dfs.namenode.randomize-block-locations-per-block</name>
+ <value>false</value>
+ <description>When fetching replica locations of a block, the replicas
+ are sorted based on network distance. This configuration parameter
+ determines whether the replicas at the same network distance are randomly
+ shuffled. By default, this is false, such that repeated requests for a block's
+ replicas always result in the same order. This potentially improves page cache
+ behavior. However, for some network topologies, it is desirable to shuffle this
+ order for better load balancing.
+ </description>
+</property>
+
</configuration>
Propchange: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1609878-1613507
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html Fri Jul 25 20:33:09 2014
@@ -66,7 +66,6 @@
<div class="row">
<hr />
<div class="col-xs-2"><p>Hadoop, 2014.</p></div>
- <div class="col-xs-1 pull-right"><a style="color: #ddd" href="dfshealth.jsp">Legacy UI</a></div>
</div>
</div>
@@ -283,7 +282,7 @@
<tr class="danger">
<td>{name} ({xferaddr})</td>
<td>{lastContact}</td>
- <td>Dead{?decomissioned}, Decomissioned{/decomissioned}</td>
+ <td>Dead{?decommissioned}, Decommissioned{/decommissioned}</td>
<td>-</td>
<td>-</td>
<td>-</td>
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html Fri Jul 25 20:33:09 2014
@@ -24,6 +24,29 @@
<title>Browsing HDFS</title>
</head>
<body>
+
+ <header class="navbar navbar-inverse bs-docs-nav" role="banner">
+ <div class="container">
+ <div class="navbar-header">
+ <div class="navbar-brand">Hadoop</div>
+ </div>
+
+ <ul class="nav navbar-nav" id="ui-tabs">
+ <li><a href="dfshealth.html#tab-overview">Overview</a></li>
+ <li><a href="dfshealth.html#tab-datanode">Datanodes</a></li>
+ <li><a href="dfshealth.html#tab-snapshot">Snapshot</a></li>
+ <li><a href="dfshealth.html#tab-startup-progress">Startup Progress</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Utilities <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="#">Browse the file system</a></li>
+ <li><a href="logs">Logs</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </header>
+
<div class="modal" id="file-info" tabindex="-1" role="dialog" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
@@ -73,6 +96,12 @@
</div>
<br />
<div id="panel"></div>
+
+ <div class="row">
+ <hr />
+ <div class="col-xs-2"><p>Hadoop, 2014.</p></div>
+ </div>
+
</div>
<script type="text/x-dust-template" id="tmpl-explorer">
@@ -126,7 +155,5 @@
</script><script type="text/javascript" src="/static/dfs-dust.js">
</script><script type="text/javascript" src="explorer.js">
</script>
- <hr />
- <p>Hadoop, 2014.</p>
</body>
</html>
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html Fri Jul 25 20:33:09 2014
@@ -18,18 +18,7 @@
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
-<meta http-equiv="REFRESH" content="1;url=dfshealth.jsp" />
+<meta http-equiv="REFRESH" content="0;url=dfshealth.html" />
<title>Hadoop Administration</title>
</head>
-<body>
-<script type="text/javascript">
-//<![CDATA[
-window.location.href='dfshealth.html';
-//]]>
-</script>
-<h1>Hadoop Administration</h1>
-<ul>
-<li><a href="dfshealth.jsp">DFS Health/Status</a></li>
-</ul>
-</body>
</html>
Propchange: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1603348-1613507
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html Fri Jul 25 20:33:09 2014
@@ -21,15 +21,4 @@
<meta http-equiv="REFRESH" content="0;url=status.html" />
<title>Hadoop Administration</title>
</head>
-<body>
-<script type="text/javascript">
-//<![CDATA[
-window.location.href='status.html';
-//]]>
-</script>
-<h1>Hadoop Administration</h1>
-<ul>
- <li><a href="status.jsp">Status</a></li>
-</ul>
-</body>
</html>
\ No newline at end of file
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm Fri Jul 25 20:33:09 2014
@@ -416,8 +416,8 @@ HDFS High Availability Using the Quorum
After all of the necessary configuration options have been set, you must
start the JournalNode daemons on the set of machines where they will run. This
- can be done by running the command "<hdfs-daemon.sh journalnode>" and waiting
- for the daemon to start on each of the relevant machines.
+ can be done by running the command "<hadoop-daemon.sh start journalnode>" and
+ waiting for the daemon to start on each of the relevant machines.
Once the JournalNodes have been started, one must initially synchronize the
two HA NameNodes' on-disk metadata.
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java Fri Jul 25 20:33:09 2014
@@ -33,9 +33,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
@@ -48,6 +50,7 @@ import org.apache.hadoop.hdfs.shortcircu
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -192,7 +195,8 @@ public class BlockReaderTestUtil {
setAllowShortCircuitLocalReads(true).
setRemotePeerFactory(new RemotePeerFactory() {
@Override
- public Peer newConnectedPeer(InetSocketAddress addr)
+ public Peer newConnectedPeer(InetSocketAddress addr,
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.
@@ -251,4 +255,4 @@ public class BlockReaderTestUtil {
LogManager.getLogger(DataNode.class.getName()).setLevel(
Level.TRACE);
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Fri Jul 25 20:33:09 2014
@@ -380,7 +380,7 @@ public class DFSTestUtil {
*/
public static void waitForReplication(MiniDFSCluster cluster, ExtendedBlock b,
int racks, int replicas, int neededReplicas)
- throws IOException, TimeoutException, InterruptedException {
+ throws TimeoutException, InterruptedException {
int curRacks = 0;
int curReplicas = 0;
int curNeededReplicas = 0;
@@ -414,7 +414,7 @@ public class DFSTestUtil {
*/
public static void waitCorruptReplicas(FileSystem fs, FSNamesystem ns,
Path file, ExtendedBlock b, int corruptRepls)
- throws IOException, TimeoutException, InterruptedException {
+ throws TimeoutException, InterruptedException {
int count = 0;
final int ATTEMPTS = 50;
int repls = ns.getBlockManager().numCorruptReplicas(b.getLocalBlock());
@@ -839,7 +839,8 @@ public class DFSTestUtil {
// send the request
new Sender(out).transferBlock(b, new Token<BlockTokenIdentifier>(),
- dfsClient.clientName, new DatanodeInfo[]{datanodes[1]});
+ dfsClient.clientName, new DatanodeInfo[]{datanodes[1]},
+ new StorageType[]{StorageType.DEFAULT});
out.flush();
return BlockOpResponseProto.parseDelimitedFrom(in);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Fri Jul 25 20:33:09 2014
@@ -19,12 +19,15 @@ package org.apache.hadoop.hdfs;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
@@ -660,73 +663,81 @@ public class MiniDFSCluster {
boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays)
throws IOException {
- ExitUtil.disableSystemExit();
+ boolean success = false;
+ try {
+ ExitUtil.disableSystemExit();
- synchronized (MiniDFSCluster.class) {
- instanceId = instanceCount++;
- }
+ synchronized (MiniDFSCluster.class) {
+ instanceId = instanceCount++;
+ }
- this.conf = conf;
- base_dir = new File(determineDfsBaseDir());
- data_dir = new File(base_dir, "data");
- this.waitSafeMode = waitSafeMode;
- this.checkExitOnShutdown = checkExitOnShutdown;
-
- int replication = conf.getInt(DFS_REPLICATION_KEY, 3);
- conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes));
- int safemodeExtension = conf.getInt(
- DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0);
- conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension);
- conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second
- conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- StaticMapping.class, DNSToSwitchMapping.class);
-
- // In an HA cluster, in order for the StandbyNode to perform checkpoints,
- // it needs to know the HTTP port of the Active. So, if ephemeral ports
- // are chosen, disable checkpoints for the test.
- if (!nnTopology.allHttpPortsSpecified() &&
- nnTopology.isHA()) {
- LOG.info("MiniDFSCluster disabling checkpointing in the Standby node " +
- "since no HTTP ports have been specified.");
- conf.setBoolean(DFS_HA_STANDBY_CHECKPOINTS_KEY, false);
- }
- if (!nnTopology.allIpcPortsSpecified() &&
- nnTopology.isHA()) {
- LOG.info("MiniDFSCluster disabling log-roll triggering in the "
- + "Standby node since no IPC ports have been specified.");
- conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
- }
+ this.conf = conf;
+ base_dir = new File(determineDfsBaseDir());
+ data_dir = new File(base_dir, "data");
+ this.waitSafeMode = waitSafeMode;
+ this.checkExitOnShutdown = checkExitOnShutdown;
+
+ int replication = conf.getInt(DFS_REPLICATION_KEY, 3);
+ conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes));
+ int safemodeExtension = conf.getInt(
+ DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0);
+ conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension);
+ conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second
+ conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ StaticMapping.class, DNSToSwitchMapping.class);
+
+ // In an HA cluster, in order for the StandbyNode to perform checkpoints,
+ // it needs to know the HTTP port of the Active. So, if ephemeral ports
+ // are chosen, disable checkpoints for the test.
+ if (!nnTopology.allHttpPortsSpecified() &&
+ nnTopology.isHA()) {
+ LOG.info("MiniDFSCluster disabling checkpointing in the Standby node " +
+ "since no HTTP ports have been specified.");
+ conf.setBoolean(DFS_HA_STANDBY_CHECKPOINTS_KEY, false);
+ }
+ if (!nnTopology.allIpcPortsSpecified() &&
+ nnTopology.isHA()) {
+ LOG.info("MiniDFSCluster disabling log-roll triggering in the "
+ + "Standby node since no IPC ports have been specified.");
+ conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
+ }
- federation = nnTopology.isFederated();
- try {
- createNameNodesAndSetConf(
- nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
- enableManagedDfsDirsRedundancy,
- format, startOpt, clusterId, conf);
- } catch (IOException ioe) {
- LOG.error("IOE creating namenodes. Permissions dump:\n" +
- createPermissionsDiagnosisString(data_dir));
- throw ioe;
- }
- if (format) {
- if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) {
- throw new IOException("Cannot remove data directory: " + data_dir +
+ federation = nnTopology.isFederated();
+ try {
+ createNameNodesAndSetConf(
+ nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
+ enableManagedDfsDirsRedundancy,
+ format, startOpt, clusterId, conf);
+ } catch (IOException ioe) {
+ LOG.error("IOE creating namenodes. Permissions dump:\n" +
createPermissionsDiagnosisString(data_dir));
+ throw ioe;
+ }
+ if (format) {
+ if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) {
+ throw new IOException("Cannot remove data directory: " + data_dir +
+ createPermissionsDiagnosisString(data_dir));
+ }
}
- }
- if (startOpt == StartupOption.RECOVER) {
- return;
- }
+ if (startOpt == StartupOption.RECOVER) {
+ return;
+ }
- // Start the DataNodes
- startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs,
- dnStartOpt != null ? dnStartOpt : startOpt,
- racks, hosts, simulatedCapacities, setupHostsFile,
- checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
- waitClusterUp();
- //make sure ProxyUsers uses the latest conf
- ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+ // Start the DataNodes
+ startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs,
+ dnStartOpt != null ? dnStartOpt : startOpt,
+ racks, hosts, simulatedCapacities, setupHostsFile,
+ checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
+ waitClusterUp();
+ //make sure ProxyUsers uses the latest conf
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+ success = true;
+ } finally {
+ if (!success) {
+ shutdown();
+ }
+ }
}
/**
@@ -1308,15 +1319,42 @@ public class MiniDFSCluster {
}
SecureResources secureResources = null;
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled() &&
+ conf.get(DFS_DATA_TRANSFER_PROTECTION_KEY) == null) {
try {
secureResources = SecureDataNodeStarter.getSecureResources(dnConf);
} catch (Exception ex) {
ex.printStackTrace();
}
}
- DataNode dn = DataNode.instantiateDataNode(dnArgs, dnConf,
- secureResources);
+ final int maxRetriesOnSasl = conf.getInt(
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY,
+ IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_DEFAULT);
+ int numRetries = 0;
+ DataNode dn = null;
+ while (true) {
+ try {
+ dn = DataNode.instantiateDataNode(dnArgs, dnConf,
+ secureResources);
+ break;
+ } catch (IOException e) {
+ // Work around issue testing security where rapidly starting multiple
+ // DataNodes using the same principal gets rejected by the KDC as a
+ // replay attack.
+ if (UserGroupInformation.isSecurityEnabled() &&
+ numRetries < maxRetriesOnSasl) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ ++numRetries;
+ continue;
+ }
+ throw e;
+ }
+ }
if(dn == null)
throw new IOException("Cannot start DataNode in "
+ dnConf.get(DFS_DATANODE_DATA_DIR_KEY));
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Fri Jul 25 20:33:09 2014
@@ -2653,6 +2653,75 @@ public class TestDFSShell {
}
}
+ /*
+ * 1. Test that CLI throws an exception and returns non-0 when user does
+ * not have permission to read an xattr.
+ * 2. Test that CLI throws an exception and returns non-0 when a non-existent
+ * xattr is requested.
+ */
+ @Test (timeout = 120000)
+ public void testGetFAttrErrors() throws Exception {
+ final UserGroupInformation user = UserGroupInformation.
+ createUserForTesting("user", new String[] {"mygroup"});
+ MiniDFSCluster cluster = null;
+ PrintStream bakErr = null;
+ try {
+ final Configuration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+
+ final FileSystem fs = cluster.getFileSystem();
+ final Path p = new Path("/foo");
+ fs.mkdirs(p);
+ bakErr = System.err;
+
+ final FsShell fshell = new FsShell(conf);
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ System.setErr(new PrintStream(out));
+
+ // No permission for "other".
+ fs.setPermission(p, new FsPermission((short) 0700));
+
+ {
+ final int ret = ToolRunner.run(fshell, new String[] {
+ "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"});
+ assertEquals("Returned should be 0", 0, ret);
+ out.reset();
+ }
+
+ user.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ int ret = ToolRunner.run(fshell, new String[] {
+ "-getfattr", "-n", "user.a1", "/foo"});
+ String str = out.toString();
+ assertTrue("xattr value was incorrectly returned",
+ str.indexOf("1234") == -1);
+ out.reset();
+ return null;
+ }
+ });
+
+ {
+ final int ret = ToolRunner.run(fshell, new String[]{
+ "-getfattr", "-n", "user.nonexistent", "/foo"});
+ String str = out.toString();
+ assertTrue("xattr value was incorrectly returned",
+ str.indexOf(
+ "getfattr: At least one of the attributes provided was not found")
+ >= 0);
+ out.reset();
+ }
+ } finally {
+ if (bakErr != null) {
+ System.setErr(bakErr);
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
/**
* Test that the server trash configuration is respected when
* the client configuration is not set.
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Fri Jul 25 20:33:09 2014
@@ -70,6 +70,9 @@ public class TestDFSUpgradeFromImage {
private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt";
private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz";
private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz";
+ private static final String HADOOP1_RESERVED_IMAGE = "hadoop-1-reserved.tgz";
+ private static final String HADOOP023_RESERVED_IMAGE =
+ "hadoop-0.23-reserved.tgz";
private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz";
private static class ReferenceFileInfo {
@@ -326,6 +329,140 @@ public class TestDFSUpgradeFromImage {
}
/**
+ * Test upgrade from a branch-1.2 image with reserved paths
+ */
+ @Test
+ public void testUpgradeFromRel1ReservedImage() throws Exception {
+ unpackStorage(HADOOP1_RESERVED_IMAGE);
+ MiniDFSCluster cluster = null;
+ // Try it once without setting the upgrade flag to ensure it fails
+ final Configuration conf = new Configuration();
+ // Try it again with a custom rename string
+ try {
+ FSImageFormat.setRenameReservedPairs(
+ ".snapshot=.user-snapshot," +
+ ".reserved=.my-reserved");
+ cluster =
+ new MiniDFSCluster.Builder(conf)
+ .format(false)
+ .startupOption(StartupOption.UPGRADE)
+ .numDataNodes(0).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ // Make sure the paths were renamed as expected
+ // Also check that paths are present after a restart, checks that the
+ // upgraded fsimage has the same state.
+ final String[] expected = new String[] {
+ "/.my-reserved",
+ "/.user-snapshot",
+ "/.user-snapshot/.user-snapshot",
+ "/.user-snapshot/open",
+ "/dir1",
+ "/dir1/.user-snapshot",
+ "/dir2",
+ "/dir2/.user-snapshot",
+ "/user",
+ "/user/andrew",
+ "/user/andrew/.user-snapshot",
+ };
+ for (int i=0; i<2; i++) {
+ // Restart the second time through this loop
+ if (i==1) {
+ cluster.finalizeCluster(conf);
+ cluster.restartNameNode(true);
+ }
+ ArrayList<Path> toList = new ArrayList<Path>();
+ toList.add(new Path("/"));
+ ArrayList<String> found = new ArrayList<String>();
+ while (!toList.isEmpty()) {
+ Path p = toList.remove(0);
+ FileStatus[] statuses = dfs.listStatus(p);
+ for (FileStatus status: statuses) {
+ final String path = status.getPath().toUri().getPath();
+ System.out.println("Found path " + path);
+ found.add(path);
+ if (status.isDirectory()) {
+ toList.add(status.getPath());
+ }
+ }
+ }
+ for (String s: expected) {
+ assertTrue("Did not find expected path " + s, found.contains(s));
+ }
+ assertEquals("Found an unexpected path while listing filesystem",
+ found.size(), expected.length);
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test upgrade from a 0.23.11 image with reserved paths
+ */
+ @Test
+ public void testUpgradeFromRel023ReservedImage() throws Exception {
+ unpackStorage(HADOOP023_RESERVED_IMAGE);
+ MiniDFSCluster cluster = null;
+ // Try it once without setting the upgrade flag to ensure it fails
+ final Configuration conf = new Configuration();
+ // Try it again with a custom rename string
+ try {
+ FSImageFormat.setRenameReservedPairs(
+ ".snapshot=.user-snapshot," +
+ ".reserved=.my-reserved");
+ cluster =
+ new MiniDFSCluster.Builder(conf)
+ .format(false)
+ .startupOption(StartupOption.UPGRADE)
+ .numDataNodes(0).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ // Make sure the paths were renamed as expected
+ // Also check that paths are present after a restart, checks that the
+ // upgraded fsimage has the same state.
+ final String[] expected = new String[] {
+ "/.user-snapshot",
+ "/dir1",
+ "/dir1/.user-snapshot",
+ "/dir2",
+ "/dir2/.user-snapshot"
+ };
+ for (int i=0; i<2; i++) {
+ // Restart the second time through this loop
+ if (i==1) {
+ cluster.finalizeCluster(conf);
+ cluster.restartNameNode(true);
+ }
+ ArrayList<Path> toList = new ArrayList<Path>();
+ toList.add(new Path("/"));
+ ArrayList<String> found = new ArrayList<String>();
+ while (!toList.isEmpty()) {
+ Path p = toList.remove(0);
+ FileStatus[] statuses = dfs.listStatus(p);
+ for (FileStatus status: statuses) {
+ final String path = status.getPath().toUri().getPath();
+ System.out.println("Found path " + path);
+ found.add(path);
+ if (status.isDirectory()) {
+ toList.add(status.getPath());
+ }
+ }
+ }
+ for (String s: expected) {
+ assertTrue("Did not find expected path " + s, found.contains(s));
+ }
+ assertEquals("Found an unexpected path while listing filesystem",
+ found.size(), expected.length);
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
* Test upgrade from 2.0 image with a variety of .snapshot and .reserved
* paths to test renaming on upgrade
*/
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java Fri Jul 25 20:33:09 2014
@@ -125,17 +125,16 @@ public class TestDataTransferProtocol {
throw eof;
}
- LOG.info("Received: " +new String(retBuf));
- LOG.info("Expected: " + StringUtils.byteToHexString(recvBuf.toByteArray()));
+ String received = StringUtils.byteToHexString(retBuf);
+ String expected = StringUtils.byteToHexString(recvBuf.toByteArray());
+ LOG.info("Received: " + received);
+ LOG.info("Expected: " + expected);
if (eofExpected) {
throw new IOException("Did not recieve IOException when an exception " +
"is expected while reading from " + datanode);
}
-
- byte[] needed = recvBuf.toByteArray();
- assertEquals(StringUtils.byteToHexString(needed),
- StringUtils.byteToHexString(retBuf));
+ assertEquals(expected, received);
} finally {
IOUtils.closeSocket(sock);
}
@@ -184,10 +183,7 @@ public class TestDataTransferProtocol {
String description, Boolean eofExcepted) throws IOException {
sendBuf.reset();
recvBuf.reset();
- sender.writeBlock(block, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- new DatanodeInfo[1], null, stage,
- 0, block.getNumBytes(), block.getNumBytes(), newGS,
- DEFAULT_CHECKSUM, CachingStrategy.newDefaultStrategy());
+ writeBlock(block, stage, newGS, DEFAULT_CHECKSUM);
if (eofExcepted) {
sendResponse(Status.ERROR, null, null, recvOut);
sendRecvData(description, true);
@@ -343,10 +339,7 @@ public class TestDataTransferProtocol {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
try {
cluster.waitActive();
- DFSClient dfsClient = new DFSClient(
- new InetSocketAddress("localhost", cluster.getNameNodePort()),
- conf);
- datanode = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
+ datanode = cluster.getFileSystem().getDataNodeStats(DatanodeReportType.LIVE)[0];
dnAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
FileSystem fileSys = cluster.getFileSystem();
@@ -381,23 +374,14 @@ public class TestDataTransferProtocol {
DataChecksum badChecksum = Mockito.spy(DEFAULT_CHECKSUM);
Mockito.doReturn(-1).when(badChecksum).getBytesPerChecksum();
- sender.writeBlock(new ExtendedBlock(poolId, newBlockId),
- BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- new DatanodeInfo[1], null,
- BlockConstructionStage.PIPELINE_SETUP_CREATE,
- 0, 0L, 0L, 0L,
- badChecksum, CachingStrategy.newDefaultStrategy());
+ writeBlock(poolId, newBlockId, badChecksum);
recvBuf.reset();
sendResponse(Status.ERROR, null, null, recvOut);
sendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.reset();
recvBuf.reset();
- sender.writeBlock(new ExtendedBlock(poolId, ++newBlockId),
- BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- new DatanodeInfo[1], null,
- BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0L, 0L, 0L,
- DEFAULT_CHECKSUM, CachingStrategy.newDefaultStrategy());
+ writeBlock(poolId, ++newBlockId, DEFAULT_CHECKSUM);
PacketHeader hdr = new PacketHeader(
4, // size of packet
@@ -416,11 +400,7 @@ public class TestDataTransferProtocol {
// test for writing a valid zero size block
sendBuf.reset();
recvBuf.reset();
- sender.writeBlock(new ExtendedBlock(poolId, ++newBlockId),
- BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- new DatanodeInfo[1], null,
- BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0L, 0L, 0L,
- DEFAULT_CHECKSUM, CachingStrategy.newDefaultStrategy());
+ writeBlock(poolId, ++newBlockId, DEFAULT_CHECKSUM);
hdr = new PacketHeader(
8, // size of packet
@@ -532,4 +512,18 @@ public class TestDataTransferProtocol {
assertTrue(hdr.sanityCheck(99));
assertFalse(hdr.sanityCheck(100));
}
+
+ void writeBlock(String poolId, long blockId, DataChecksum checksum) throws IOException {
+ writeBlock(new ExtendedBlock(poolId, blockId),
+ BlockConstructionStage.PIPELINE_SETUP_CREATE, 0L, checksum);
+ }
+
+ void writeBlock(ExtendedBlock block, BlockConstructionStage stage,
+ long newGS, DataChecksum checksum) throws IOException {
+ sender.writeBlock(block, StorageType.DEFAULT,
+ BlockTokenSecretManager.DUMMY_TOKEN, "cl",
+ new DatanodeInfo[1], new StorageType[1], null, stage,
+ 0, block.getNumBytes(), block.getNumBytes(), newGS,
+ checksum, CachingStrategy.newDefaultStrategy());
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java Fri Jul 25 20:33:09 2014
@@ -53,6 +53,8 @@ public class TestDatanodeConfig {
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, 0);
conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "localhost:0");
+ conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "localhost:0");
+ conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "localhost:0");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java Fri Jul 25 20:33:09 2014
@@ -30,6 +30,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -79,6 +81,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -97,6 +100,8 @@ public class TestFileCreation {
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
}
+ private static final String RPC_DETAILED_METRICS =
+ "RpcDetailedActivityForPort";
static final long seed = 0xDEADBEEFL;
static final int blockSize = 8192;
@@ -371,7 +376,7 @@ public class TestFileCreation {
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
-
+
UserGroupInformation otherUgi = UserGroupInformation.createUserForTesting(
"testuser", new String[]{"testgroup"});
FileSystem fs2 = otherUgi.doAs(new PrivilegedExceptionAction<FileSystem>() {
@@ -380,12 +385,16 @@ public class TestFileCreation {
return FileSystem.get(cluster.getConfiguration(0));
}
});
-
+
+ String metricsName = RPC_DETAILED_METRICS + cluster.getNameNodePort();
+
try {
Path p = new Path("/testfile");
FSDataOutputStream stm1 = fs.create(p);
stm1.write(1);
+ assertCounter("CreateNumOps", 1L, getMetrics(metricsName));
+
// Create file again without overwrite
try {
fs2.create(p, false);
@@ -394,7 +403,9 @@ public class TestFileCreation {
GenericTestUtils.assertExceptionContains("already being created by",
abce);
}
-
+ // NameNodeProxies' createNNProxyWithClientProtocol has 5 retries.
+ assertCounter("AlreadyBeingCreatedExceptionNumOps",
+ 6L, getMetrics(metricsName));
FSDataOutputStream stm2 = fs2.create(p, true);
stm2.write(2);
stm2.close();
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestIsMethodSupported.java Fri Jul 25 20:33:09 2014
@@ -25,14 +25,16 @@ import java.net.InetSocketAddress;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.UserGroupInformation;
@@ -76,16 +78,22 @@ public class TestIsMethodSupported {
@Test
public void testNamenodeProtocol() throws IOException {
- NamenodeProtocolTranslatorPB translator =
- (NamenodeProtocolTranslatorPB) NameNodeProxies.createNonHAProxy(conf,
+ NamenodeProtocol np =
+ NameNodeProxies.createNonHAProxy(conf,
nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
true).getProxy();
- boolean exists = translator.isMethodSupported("rollEditLog");
+
+ boolean exists = RpcClientUtil.isMethodSupported(np,
+ NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(NamenodeProtocolPB.class), "rollEditLog");
+
assertTrue(exists);
- exists = translator.isMethodSupported("bogusMethod");
+ exists = RpcClientUtil.isMethodSupported(np,
+ NamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(NamenodeProtocolPB.class), "bogusMethod");
assertFalse(exists);
}
-
+
@Test
public void testDatanodeProtocol() throws IOException {
DatanodeProtocolClientSideTranslatorPB translator =
@@ -107,16 +115,18 @@ public class TestIsMethodSupported {
NetUtils.getDefaultSocketFactory(conf));
assertTrue(translator.isMethodSupported("refreshNamenodes"));
}
-
+
@Test
public void testClientNamenodeProtocol() throws IOException {
- ClientNamenodeProtocolTranslatorPB translator =
- (ClientNamenodeProtocolTranslatorPB) NameNodeProxies.createNonHAProxy(
+ ClientProtocol cp =
+ NameNodeProxies.createNonHAProxy(
conf, nnAddress, ClientProtocol.class,
UserGroupInformation.getCurrentUser(), true).getProxy();
- assertTrue(translator.isMethodSupported("mkdirs"));
+ RpcClientUtil.isMethodSupported(cp,
+ ClientNamenodeProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(ClientNamenodeProtocolPB.class), "mkdirs");
}
-
+
@Test
public void tesJournalProtocol() throws IOException {
JournalProtocolTranslatorPB translator = (JournalProtocolTranslatorPB)
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java Fri Jul 25 20:33:09 2014
@@ -550,8 +550,10 @@ public class TestPBHelper {
dnInfos[1][0] = DFSTestUtil.getLocalDatanodeInfo();
dnInfos[1][1] = DFSTestUtil.getLocalDatanodeInfo();
String[][] storageIDs = {{"s00"}, {"s10", "s11"}};
+ StorageType[][] storageTypes = {{StorageType.DEFAULT},
+ {StorageType.DEFAULT, StorageType.DEFAULT}};
BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1",
- blocks, dnInfos, storageIDs);
+ blocks, dnInfos, storageTypes, storageIDs);
BlockCommandProto bcProto = PBHelper.convert(bc);
BlockCommand bc2 = PBHelper.convert(bcProto);
assertEquals(bc.getAction(), bc2.getAction());
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java Fri Jul 25 20:33:09 2014
@@ -89,7 +89,8 @@ public class TestDelegationTokenForProxy
builder.append("127.0.1.1,");
builder.append(InetAddress.getLocalHost().getCanonicalHostName());
LOG.info("Local Ip addresses: " + builder.toString());
- conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName),
+ conf.setStrings(DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserIpConfKey(superUserShortName),
builder.toString());
}
@@ -101,7 +102,8 @@ public class TestDelegationTokenForProxy
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
config.setLong(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
- config.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER),
+ config.setStrings(DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserGroupConfKey(REAL_USER),
"group1");
config.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java Fri Jul 25 20:33:09 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.RemotePeerFactory;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -160,7 +161,8 @@ public class TestBlockTokenWithDFS {
setConfiguration(conf).
setRemotePeerFactory(new RemotePeerFactory() {
@Override
- public Peer newConnectedPeer(InetSocketAddress addr)
+ public Peer newConnectedPeer(InetSocketAddress addr,
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Fri Jul 25 20:33:09 2014
@@ -905,49 +905,46 @@ public class TestReplicationPolicy {
*/
@Test
public void testChooseReplicaToDelete() throws Exception {
- List<DatanodeDescriptor> replicaNodeList = new
- ArrayList<DatanodeDescriptor>();
- final Map<String, List<DatanodeDescriptor>> rackMap
- = new HashMap<String, List<DatanodeDescriptor>>();
+ List<DatanodeStorageInfo> replicaList = new ArrayList<DatanodeStorageInfo>();
+ final Map<String, List<DatanodeStorageInfo>> rackMap
+ = new HashMap<String, List<DatanodeStorageInfo>>();
dataNodes[0].setRemaining(4*1024*1024);
- replicaNodeList.add(dataNodes[0]);
+ replicaList.add(storages[0]);
dataNodes[1].setRemaining(3*1024*1024);
- replicaNodeList.add(dataNodes[1]);
+ replicaList.add(storages[1]);
dataNodes[2].setRemaining(2*1024*1024);
- replicaNodeList.add(dataNodes[2]);
+ replicaList.add(storages[2]);
dataNodes[5].setRemaining(1*1024*1024);
- replicaNodeList.add(dataNodes[5]);
+ replicaList.add(storages[5]);
// Refresh the last update time for all the datanodes
for (int i = 0; i < dataNodes.length; i++) {
dataNodes[i].setLastUpdate(Time.now());
}
- List<DatanodeDescriptor> first = new ArrayList<DatanodeDescriptor>();
- List<DatanodeDescriptor> second = new ArrayList<DatanodeDescriptor>();
- replicator.splitNodesWithRack(
- replicaNodeList, rackMap, first, second);
- // dataNodes[0] and dataNodes[1] are in first set as their rack has two
- // replica nodes, while datanodes[2] and dataNodes[5] are in second set.
+ List<DatanodeStorageInfo> first = new ArrayList<DatanodeStorageInfo>();
+ List<DatanodeStorageInfo> second = new ArrayList<DatanodeStorageInfo>();
+ replicator.splitNodesWithRack(replicaList, rackMap, first, second);
+ // storages[0] and storages[1] are in first set as their rack has two
+ // replica nodes, while storages[2] and dataNodes[5] are in second set.
assertEquals(2, first.size());
assertEquals(2, second.size());
- DatanodeDescriptor chosenNode = replicator.chooseReplicaToDelete(
+ DatanodeStorageInfo chosen = replicator.chooseReplicaToDelete(
null, null, (short)3, first, second);
- // Within first set, dataNodes[1] with less free space
- assertEquals(chosenNode, dataNodes[1]);
+ // Within first set, storages[1] with less free space
+ assertEquals(chosen, storages[1]);
- replicator.adjustSetsWithChosenReplica(
- rackMap, first, second, chosenNode);
+ replicator.adjustSetsWithChosenReplica(rackMap, first, second, chosen);
assertEquals(0, first.size());
assertEquals(3, second.size());
- // Within second set, dataNodes[5] with less free space
- chosenNode = replicator.chooseReplicaToDelete(
+ // Within second set, storages[5] with less free space
+ chosen = replicator.chooseReplicaToDelete(
null, null, (short)2, first, second);
- assertEquals(chosenNode, dataNodes[5]);
+ assertEquals(chosen, storages[5]);
}
/**