You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2014/08/01 03:29:54 UTC
svn commit: r1615020 [2/2] - in
/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/...
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Fri Aug 1 01:29:49 2014
@@ -2855,12 +2855,15 @@ public class BlockManager {
} else {
final String[] datanodeUuids = new String[locations.size()];
final String[] storageIDs = new String[datanodeUuids.length];
+ final StorageType[] storageTypes = new StorageType[datanodeUuids.length];
for(int i = 0; i < locations.size(); i++) {
final DatanodeStorageInfo s = locations.get(i);
datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid();
storageIDs[i] = s.getStorageID();
+ storageTypes[i] = s.getStorageType();
}
- results.add(new BlockWithLocations(block, datanodeUuids, storageIDs));
+ results.add(new BlockWithLocations(block, datanodeUuids, storageIDs,
+ storageTypes));
return block.getNumBytes();
}
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java Fri Aug 1 01:29:49 2014
@@ -312,7 +312,7 @@ public class BlockPoolSliceStorage exten
}
LOG.info("Upgrading block pool storage directory " + bpSd.getRoot()
+ ".\n old LV = " + this.getLayoutVersion() + "; old CTime = "
- + this.getCTime() + ".\n new LV = " + nsInfo.getLayoutVersion()
+ + this.getCTime() + ".\n new LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION
+ "; new CTime = " + nsInfo.getCTime());
// get <SD>/previous directory
String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath());
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Aug 1 01:29:49 2014
@@ -8461,6 +8461,29 @@ public class FSNamesystem implements Nam
}
}
+ void checkAccess(String src, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ checkOperation(OperationCategory.READ);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ src = FSDirectory.resolvePath(src, pathComponents, dir);
+ if (dir.getINode(src) == null) {
+ throw new FileNotFoundException("Path not found");
+ }
+ if (isPermissionEnabled) {
+ FSPermissionChecker pc = getPermissionChecker();
+ checkPathAccess(pc, src, mode);
+ }
+ } catch (AccessControlException e) {
+ logAuditEvent(false, "checkAccess", src);
+ throw e;
+ } finally {
+ readUnlock();
+ }
+ }
+
/**
* Default AuditLogger implementation; used when no access logger is
* defined in the config file. It can also be explicitly listed in the
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Fri Aug 1 01:29:49 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.XAttrSetFlag
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
@@ -1443,5 +1444,10 @@ class NameNodeRpcServer implements Namen
public void removeXAttr(String src, XAttr xAttr) throws IOException {
namesystem.removeXAttr(src, xAttr);
}
+
+ @Override
+ public void checkAccess(String path, FsAction mode) throws IOException {
+ namesystem.checkAccess(path, mode);
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Fri Aug 1 01:29:49 2014
@@ -56,6 +56,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -110,6 +112,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.XAttrNameParam;
import org.apache.hadoop.hdfs.web.resources.XAttrSetFlagParam;
import org.apache.hadoop.hdfs.web.resources.XAttrValueParam;
+import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.Server;
@@ -750,10 +753,12 @@ public class NamenodeWebHdfsMethods {
@QueryParam(XAttrEncodingParam.NAME) @DefaultValue(XAttrEncodingParam.DEFAULT)
final XAttrEncodingParam xattrEncoding,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT)
+ final FsActionParam fsAction
) throws IOException, InterruptedException {
return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
- renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes);
+ renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction);
}
/** Handle HTTP GET request. */
@@ -784,11 +789,13 @@ public class NamenodeWebHdfsMethods {
@QueryParam(XAttrEncodingParam.NAME) @DefaultValue(XAttrEncodingParam.DEFAULT)
final XAttrEncodingParam xattrEncoding,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT)
+ final FsActionParam fsAction
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, offset, length,
- renewer, bufferSize, xattrEncoding, excludeDatanodes);
+ renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -796,7 +803,7 @@ public class NamenodeWebHdfsMethods {
try {
return get(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, offset, length, renewer, bufferSize,
- xattrNames, xattrEncoding, excludeDatanodes);
+ xattrNames, xattrEncoding, excludeDatanodes, fsAction);
} finally {
reset();
}
@@ -817,7 +824,8 @@ public class NamenodeWebHdfsMethods {
final BufferSizeParam bufferSize,
final List<XAttrNameParam> xattrNames,
final XAttrEncodingParam xattrEncoding,
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ final FsActionParam fsAction
) throws IOException, URISyntaxException {
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final NamenodeProtocols np = getRPCServer(namenode);
@@ -914,6 +922,10 @@ public class NamenodeWebHdfsMethods {
final String js = JsonUtil.toJsonString(xAttrs);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
+ case CHECKACCESS: {
+ np.checkAccess(fullpath, FsAction.getFsAction(fsAction.getValue()));
+ return Response.ok().build();
+ }
default:
throw new UnsupportedOperationException(op + " is not supported");
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java Fri Aug 1 01:29:49 2014
@@ -17,10 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
-import java.util.Arrays;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
/**
@@ -39,12 +38,15 @@ public class BlocksWithLocations {
final Block block;
final String[] datanodeUuids;
final String[] storageIDs;
+ final StorageType[] storageTypes;
/** constructor */
- public BlockWithLocations(Block block, String[] datanodeUuids, String[] storageIDs) {
+ public BlockWithLocations(Block block, String[] datanodeUuids,
+ String[] storageIDs, StorageType[] storageTypes) {
this.block = block;
this.datanodeUuids = datanodeUuids;
this.storageIDs = storageIDs;
+ this.storageTypes = storageTypes;
}
/** get the block */
@@ -61,7 +63,12 @@ public class BlocksWithLocations {
public String[] getStorageIDs() {
return storageIDs;
}
-
+
+ /** @return the storage types */
+ public StorageType[] getStorageTypes() {
+ return storageTypes;
+ }
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
@@ -70,12 +77,18 @@ public class BlocksWithLocations {
return b.append("[]").toString();
}
- b.append(storageIDs[0]).append('@').append(datanodeUuids[0]);
+ appendString(0, b.append("["));
for(int i = 1; i < datanodeUuids.length; i++) {
- b.append(", ").append(storageIDs[i]).append("@").append(datanodeUuids[i]);
+ appendString(i, b.append(","));
}
return b.append("]").toString();
}
+
+ private StringBuilder appendString(int i, StringBuilder b) {
+ return b.append("[").append(storageTypes[i]).append("]")
+ .append(storageIDs[i])
+ .append("@").append(datanodeUuids[i]);
+ }
}
private final BlockWithLocations[] blocks;
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java Fri Aug 1 01:29:49 2014
@@ -29,8 +29,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
-import com.sun.org.apache.xml.internal.serialize.OutputFormat;
-import com.sun.org.apache.xml.internal.serialize.XMLSerializer;
+import org.apache.xml.serialize.OutputFormat;
+import org.apache.xml.serialize.XMLSerializer;
/**
* An XmlEditsVisitor walks over an EditLog structure and writes out
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java Fri Aug 1 01:29:49 2014
@@ -37,7 +37,7 @@ import com.google.common.base.Preconditi
public class EnumCounters<E extends Enum<E>> {
/** The class of the enum. */
private final Class<E> enumClass;
- /** The counter array, counters[i] corresponds to the enumConstants[i]. */
+ /** An array of longs corresponding to the enum type. */
private final long[] counters;
/**
@@ -75,6 +75,13 @@ public class EnumCounters<E extends Enum
}
}
+ /** Reset all counters to zero. */
+ public final void reset() {
+ for(int i = 0; i < counters.length; i++) {
+ this.counters[i] = 0L;
+ }
+ }
+
/** Add the given value to counter e. */
public final void add(final E e, final long value) {
counters[e.ordinal()] += value;
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Fri Aug 1 01:29:49 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -1357,6 +1358,12 @@ public class WebHdfsFileSystem extends F
}
@Override
+ public void access(final Path path, final FsAction mode) throws IOException {
+ final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
+ new FsPathRunner(op, path, new FsActionParam(mode)).run();
+ }
+
+ @Override
public ContentSummary getContentSummary(final Path p) throws IOException {
statistics.incrementReadOps(1);
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java Fri Aug 1 01:29:49 2014
@@ -39,7 +39,9 @@ public class GetOpParam extends HttpOpPa
GETXATTRS(false, HttpURLConnection.HTTP_OK),
LISTXATTRS(false, HttpURLConnection.HTTP_OK),
- NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED);
+ NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
+
+ CHECKACCESS(false, HttpURLConnection.HTTP_OK);
final boolean redirect;
final int expectedHttpResponseCode;
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Fri Aug 1 01:29:49 2014
@@ -654,6 +654,14 @@ message DeleteSnapshotRequestProto {
message DeleteSnapshotResponseProto { // void response
}
+message CheckAccessRequestProto {
+ required string path = 1;
+ required AclEntryProto.FsActionProto mode = 2;
+}
+
+message CheckAccessResponseProto { // void response
+}
+
service ClientNamenodeProtocol {
rpc getBlockLocations(GetBlockLocationsRequestProto)
returns(GetBlockLocationsResponseProto);
@@ -783,4 +791,6 @@ service ClientNamenodeProtocol {
returns(ListXAttrsResponseProto);
rpc removeXAttr(RemoveXAttrRequestProto)
returns(RemoveXAttrResponseProto);
+ rpc checkAccess(CheckAccessRequestProto)
+ returns(CheckAccessResponseProto);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Fri Aug 1 01:29:49 2014
@@ -405,6 +405,7 @@ message BlockWithLocationsProto {
required BlockProto block = 1; // Block
repeated string datanodeUuids = 2; // Datanodes with replicas of the block
repeated string storageUuids = 3; // Storages with replicas of the block
+ repeated StorageTypeProto storageTypes = 4;
}
/**
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Fri Aug 1 01:29:49 2014
@@ -82,6 +82,9 @@ WebHDFS REST API
* {{{List all XAttrs}<<<LISTXATTRS>>>}}
(see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.listXAttrs)
+ * {{{Check access}<<<CHECKACCESS>>>}}
+ (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.access)
+
* HTTP PUT
* {{{Create and Write to a File}<<<CREATE>>>}}
@@ -927,6 +930,28 @@ Transfer-Encoding: chunked
{{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.getAclStatus
+** {Check access}
+
+ * Submit a HTTP GET request.
+
++---------------------------------
+curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CHECKACCESS
+ &fsaction=<FSACTION>
++---------------------------------
+
+ The client receives a response with zero content length:
+
++---------------------------------
+HTTP/1.1 200 OK
+Content-Length: 0
++---------------------------------
+
+ []
+
+ See also:
+ {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.access
+
+
* {Extended Attributes(XAttrs) Operations}
** {Set XAttr}
@@ -2166,6 +2191,25 @@ var tokenProperties =
{{Proxy Users}}
+** {Fs Action}
+
+*----------------+-------------------------------------------------------------------+
+|| Name | <<<fsaction>>> |
+*----------------+-------------------------------------------------------------------+
+|| Description | File system operation read/write/execute |
+*----------------+-------------------------------------------------------------------+
+|| Type | String |
+*----------------+-------------------------------------------------------------------+
+|| Default Value | null (an invalid value) |
+*----------------+-------------------------------------------------------------------+
+|| Valid Values | Strings matching regex pattern \"[rwx-]\{3\}\" |
+*----------------+-------------------------------------------------------------------+
+|| Syntax | \"[rwx-]\{3\}\" |
+*----------------+-------------------------------------------------------------------+
+
+ See also:
+ {{{Check access}<<<CHECKACCESS>>>}},
+
** {Group}
*----------------+-------------------------------------------------------------------+
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java Fri Aug 1 01:29:49 2014
@@ -47,7 +47,6 @@ import org.mockito.Mockito;
public class TestGenericRefresh {
private static MiniDFSCluster cluster;
private static Configuration config;
- private static final int NNPort = 54222;
private static RefreshHandler firstHandler;
private static RefreshHandler secondHandler;
@@ -57,8 +56,8 @@ public class TestGenericRefresh {
config = new Configuration();
config.set("hadoop.security.authorization", "true");
- FileSystem.setDefaultUri(config, "hdfs://localhost:" + NNPort);
- cluster = new MiniDFSCluster.Builder(config).nameNodePort(NNPort).build();
+ FileSystem.setDefaultUri(config, "hdfs://localhost:0");
+ cluster = new MiniDFSCluster.Builder(config).build();
cluster.waitActive();
}
@@ -103,7 +102,8 @@ public class TestGenericRefresh {
@Test
public void testInvalidIdentifier() throws Exception {
DFSAdmin admin = new DFSAdmin(config);
- String [] args = new String[]{"-refresh", "localhost:" + NNPort, "unregisteredIdentity"};
+ String [] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "unregisteredIdentity"};
int exitCode = admin.run(args);
assertEquals("DFSAdmin should fail due to no handler registered", -1, exitCode);
}
@@ -111,7 +111,8 @@ public class TestGenericRefresh {
@Test
public void testValidIdentifier() throws Exception {
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "firstHandler"};
+ String[] args = new String[]{"-refresh",
+ "localhost:" + cluster.getNameNodePort(), "firstHandler"};
int exitCode = admin.run(args);
assertEquals("DFSAdmin should succeed", 0, exitCode);
@@ -124,11 +125,13 @@ public class TestGenericRefresh {
@Test
public void testVariableArgs() throws Exception {
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "secondHandler", "one"};
+ String[] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "secondHandler", "one"};
int exitCode = admin.run(args);
assertEquals("DFSAdmin should return 2", 2, exitCode);
- exitCode = admin.run(new String[]{"-refresh", "localhost:" + NNPort, "secondHandler", "one", "two"});
+ exitCode = admin.run(new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "secondHandler", "one", "two"});
assertEquals("DFSAdmin should now return 3", 3, exitCode);
Mockito.verify(secondHandler).handleRefresh("secondHandler", new String[]{"one"});
@@ -141,7 +144,8 @@ public class TestGenericRefresh {
// And now this should fail
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "firstHandler"};
+ String[] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "firstHandler"};
int exitCode = admin.run(args);
assertEquals("DFSAdmin should return -1", -1, exitCode);
}
@@ -161,7 +165,8 @@ public class TestGenericRefresh {
// this should trigger both
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "sharedId", "one"};
+ String[] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "sharedId", "one"};
int exitCode = admin.run(args);
assertEquals(-1, exitCode); // -1 because one of the responses is unregistered
@@ -189,7 +194,8 @@ public class TestGenericRefresh {
// We refresh both
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "shared"};
+ String[] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "shared"};
int exitCode = admin.run(args);
assertEquals(-1, exitCode); // We get -1 because of our logic for melding non-zero return codes
@@ -215,7 +221,8 @@ public class TestGenericRefresh {
RefreshRegistry.defaultRegistry().register("exceptional", otherExceptionalHandler);
DFSAdmin admin = new DFSAdmin(config);
- String[] args = new String[]{"-refresh", "localhost:" + NNPort, "exceptional"};
+ String[] args = new String[]{"-refresh", "localhost:" +
+ cluster.getNameNodePort(), "exceptional"};
int exitCode = admin.run(args);
assertEquals(-1, exitCode); // Exceptions result in a -1
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java Fri Aug 1 01:29:49 2014
@@ -24,6 +24,8 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.net.BindException;
+import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -42,24 +44,42 @@ public class TestRefreshCallQueue {
private FileSystem fs;
static int mockQueueConstructions;
static int mockQueuePuts;
- private static final int NNPort = 54222;
- private static String CALLQUEUE_CONFIG_KEY = "ipc." + NNPort + ".callqueue.impl";
+ private String callQueueConfigKey = "";
+ private final Random rand = new Random();
@Before
public void setUp() throws Exception {
// We want to count additional events, so we reset here
mockQueueConstructions = 0;
mockQueuePuts = 0;
+ int portRetries = 5;
+ int nnPort;
- config = new Configuration();
- config.setClass(CALLQUEUE_CONFIG_KEY,
- MockCallQueue.class, BlockingQueue.class);
- config.set("hadoop.security.authorization", "true");
-
- FileSystem.setDefaultUri(config, "hdfs://localhost:" + NNPort);
- fs = FileSystem.get(config);
- cluster = new MiniDFSCluster.Builder(config).nameNodePort(NNPort).build();
- cluster.waitActive();
+ for (; portRetries > 0; --portRetries) {
+ // Pick a random port in the range [30000,60000).
+ nnPort = 30000 + rand.nextInt(30000);
+ config = new Configuration();
+ callQueueConfigKey = "ipc." + nnPort + ".callqueue.impl";
+ config.setClass(callQueueConfigKey,
+ MockCallQueue.class, BlockingQueue.class);
+ config.set("hadoop.security.authorization", "true");
+
+ FileSystem.setDefaultUri(config, "hdfs://localhost:" + nnPort);
+ fs = FileSystem.get(config);
+
+ try {
+ cluster = new MiniDFSCluster.Builder(config).nameNodePort(nnPort).build();
+ cluster.waitActive();
+ break;
+ } catch (BindException be) {
+ // Retry with a different port number.
+ }
+ }
+
+ if (portRetries == 0) {
+ // Bail if we get very unlucky with our choice of ports.
+ fail("Failed to pick an ephemeral port for the NameNode RPC server.");
+ }
}
@After
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java Fri Aug 1 01:29:49 2014
@@ -20,8 +20,11 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -36,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
@@ -421,6 +425,79 @@ public class TestDFSPermission {
}
}
+ @Test
+ public void testAccessOwner() throws IOException, InterruptedException {
+ FileSystem rootFs = FileSystem.get(conf);
+ Path p1 = new Path("/p1");
+ rootFs.mkdirs(p1);
+ rootFs.setOwner(p1, USER1_NAME, GROUP1_NAME);
+ fs = USER1.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ return FileSystem.get(conf);
+ }
+ });
+ fs.setPermission(p1, new FsPermission((short) 0444));
+ fs.access(p1, FsAction.READ);
+ try {
+ fs.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad/bad");
+ try {
+ fs.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testAccessGroupMember() throws IOException, InterruptedException {
+ FileSystem rootFs = FileSystem.get(conf);
+ Path p2 = new Path("/p2");
+ rootFs.mkdirs(p2);
+ rootFs.setOwner(p2, UserGroupInformation.getCurrentUser().getShortUserName(), GROUP1_NAME);
+ rootFs.setPermission(p2, new FsPermission((short) 0740));
+ fs = USER1.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ return FileSystem.get(conf);
+ }
+ });
+ fs.access(p2, FsAction.READ);
+ try {
+ fs.access(p2, FsAction.EXECUTE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testAccessOthers() throws IOException, InterruptedException {
+ FileSystem rootFs = FileSystem.get(conf);
+ Path p3 = new Path("/p3");
+ rootFs.mkdirs(p3);
+ rootFs.setPermission(p3, new FsPermission((short) 0774));
+ fs = USER1.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ return FileSystem.get(conf);
+ }
+ });
+ fs.access(p3, FsAction.READ);
+ try {
+ fs.access(p3, FsAction.READ_WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+ }
+
/* Check if namenode performs permission checking correctly
* for the given user for operations mkdir, open, setReplication,
* getFileInfo, isDirectory, exists, getContentLength, list, rename,
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java Fri Aug 1 01:29:49 2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -36,6 +37,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
@@ -47,6 +49,8 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
@@ -297,7 +301,8 @@ public class TestSafeMode {
* assert that they are either allowed or fail as expected.
*/
@Test
- public void testOperationsWhileInSafeMode() throws IOException {
+ public void testOperationsWhileInSafeMode() throws IOException,
+ InterruptedException {
final Path file1 = new Path("/file1");
assertFalse(dfs.setSafeMode(SafeModeAction.SAFEMODE_GET));
@@ -407,6 +412,22 @@ public class TestSafeMode {
fail("getAclStatus failed while in SM");
}
+ // Test access
+ UserGroupInformation ugiX = UserGroupInformation.createRemoteUser("userX");
+ FileSystem myfs = ugiX.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return FileSystem.get(conf);
+ }
+ });
+ myfs.access(file1, FsAction.READ);
+ try {
+ myfs.access(file1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
assertFalse("Could not leave SM",
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE));
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java Fri Aug 1 01:29:49 2014
@@ -184,8 +184,10 @@ public class TestPBHelper {
private static BlockWithLocations getBlockWithLocations(int bid) {
final String[] datanodeUuids = {"dn1", "dn2", "dn3"};
final String[] storageIDs = {"s1", "s2", "s3"};
+ final StorageType[] storageTypes = {
+ StorageType.DISK, StorageType.DISK, StorageType.DISK};
return new BlockWithLocations(new Block(bid, 0, 1),
- datanodeUuids, storageIDs);
+ datanodeUuids, storageIDs, storageTypes);
}
private void compare(BlockWithLocations locs1, BlockWithLocations locs2) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Fri Aug 1 01:29:49 2014
@@ -18,17 +18,23 @@
package org.apache.hadoop.hdfs.server.balancer;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -48,6 +54,8 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli;
+import org.apache.hadoop.hdfs.server.balancer.Balancer.Parameters;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
@@ -255,6 +263,18 @@ public class TestBalancer {
}
}
}
+
+ /**
+ * Wait until balanced: each datanode gives utilization within
+ * BALANCE_ALLOWED_VARIANCE of average
+ * @throws IOException
+ * @throws TimeoutException
+ */
+ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
+ ClientProtocol client, MiniDFSCluster cluster, Balancer.Parameters p)
+ throws IOException, TimeoutException {
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, 0);
+ }
/**
* Wait until balanced: each datanode gives utilization within
@@ -263,11 +283,17 @@ public class TestBalancer {
* @throws TimeoutException
*/
static void waitForBalancer(long totalUsedSpace, long totalCapacity,
- ClientProtocol client, MiniDFSCluster cluster)
- throws IOException, TimeoutException {
+ ClientProtocol client, MiniDFSCluster cluster, Balancer.Parameters p,
+ int expectedExcludedNodes) throws IOException, TimeoutException {
long timeout = TIMEOUT;
long failtime = (timeout <= 0L) ? Long.MAX_VALUE
: Time.now() + timeout;
+ if (!p.nodesToBeIncluded.isEmpty()) {
+ totalCapacity = p.nodesToBeIncluded.size() * CAPACITY;
+ }
+ if (!p.nodesToBeExcluded.isEmpty()) {
+ totalCapacity -= p.nodesToBeExcluded.size() * CAPACITY;
+ }
final double avgUtilization = ((double)totalUsedSpace) / totalCapacity;
boolean balanced;
do {
@@ -275,9 +301,20 @@ public class TestBalancer {
client.getDatanodeReport(DatanodeReportType.ALL);
assertEquals(datanodeReport.length, cluster.getDataNodes().size());
balanced = true;
+ int actualExcludedNodeCount = 0;
for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity();
+ if (Balancer.Util.shouldBeExcluded(p.nodesToBeExcluded, datanode)) {
+ assertTrue(nodeUtilization == 0);
+ actualExcludedNodeCount++;
+ continue;
+ }
+ if (!Balancer.Util.shouldBeIncluded(p.nodesToBeIncluded, datanode)) {
+ assertTrue(nodeUtilization == 0);
+ actualExcludedNodeCount++;
+ continue;
+ }
if (Math.abs(avgUtilization - nodeUtilization) > BALANCE_ALLOWED_VARIANCE) {
balanced = false;
if (Time.now() > failtime) {
@@ -294,6 +331,7 @@ public class TestBalancer {
break;
}
}
+ assertEquals(expectedExcludedNodes,actualExcludedNodeCount);
} while (!balanced);
}
@@ -307,22 +345,118 @@ public class TestBalancer {
}
return b.append("]").toString();
}
- /** This test start a cluster with specified number of nodes,
+ /**
+ * Class which contains information about the
+ * new nodes to be added to the cluster for balancing.
+ */
+ static abstract class NewNodeInfo {
+
+ Set<String> nodesToBeExcluded = new HashSet<String>();
+ Set<String> nodesToBeIncluded = new HashSet<String>();
+
+ abstract String[] getNames();
+ abstract int getNumberofNewNodes();
+ abstract int getNumberofIncludeNodes();
+ abstract int getNumberofExcludeNodes();
+
+ public Set<String> getNodesToBeIncluded() {
+ return nodesToBeIncluded;
+ }
+ public Set<String> getNodesToBeExcluded() {
+ return nodesToBeExcluded;
+ }
+ }
+
+ /**
+ * The host names of new nodes are specified
+ */
+ static class HostNameBasedNodes extends NewNodeInfo {
+ String[] hostnames;
+
+ public HostNameBasedNodes(String[] hostnames,
+ Set<String> nodesToBeExcluded, Set<String> nodesToBeIncluded) {
+ this.hostnames = hostnames;
+ this.nodesToBeExcluded = nodesToBeExcluded;
+ this.nodesToBeIncluded = nodesToBeIncluded;
+ }
+
+ @Override
+ String[] getNames() {
+ return hostnames;
+ }
+ @Override
+ int getNumberofNewNodes() {
+ return hostnames.length;
+ }
+ @Override
+ int getNumberofIncludeNodes() {
+ return nodesToBeIncluded.size();
+ }
+ @Override
+ int getNumberofExcludeNodes() {
+ return nodesToBeExcluded.size();
+ }
+ }
+
+ /**
+ * The number of data nodes to be started are specified.
+ * The data nodes will have same host name, but different port numbers.
+ *
+ */
+ static class PortNumberBasedNodes extends NewNodeInfo {
+ int newNodes;
+ int excludeNodes;
+ int includeNodes;
+
+ public PortNumberBasedNodes(int newNodes, int excludeNodes, int includeNodes) {
+ this.newNodes = newNodes;
+ this.excludeNodes = excludeNodes;
+ this.includeNodes = includeNodes;
+ }
+
+ @Override
+ String[] getNames() {
+ return null;
+ }
+ @Override
+ int getNumberofNewNodes() {
+ return newNodes;
+ }
+ @Override
+ int getNumberofIncludeNodes() {
+ return includeNodes;
+ }
+ @Override
+ int getNumberofExcludeNodes() {
+ return excludeNodes;
+ }
+ }
+
+ private void doTest(Configuration conf, long[] capacities, String[] racks,
+ long newCapacity, String newRack, boolean useTool) throws Exception {
+ doTest(conf, capacities, racks, newCapacity, newRack, null, useTool, false);
+ }
+
+ /** This test start a cluster with specified number of nodes,
* and fills it to be 30% full (with a single file replicated identically
* to all datanodes);
* It then adds one new empty node and starts balancing.
- *
+ *
* @param conf - configuration
* @param capacities - array of capacities of original nodes in cluster
* @param racks - array of racks for original nodes in cluster
* @param newCapacity - new node's capacity
* @param newRack - new node's rack
+ * @param nodes - information about new nodes to be started.
* @param useTool - if true run test via Cli with command-line argument
* parsing, etc. Otherwise invoke balancer API directly.
+ * @param useFile - if true, the hosts to included or excluded will be stored in a
+ * file and then later read from the file.
* @throws Exception
*/
- private void doTest(Configuration conf, long[] capacities, String[] racks,
- long newCapacity, String newRack, boolean useTool) throws Exception {
+ private void doTest(Configuration conf, long[] capacities,
+ String[] racks, long newCapacity, String newRack, NewNodeInfo nodes,
+ boolean useTool, boolean useFile) throws Exception {
LOG.info("capacities = " + long2String(capacities));
LOG.info("racks = " + Arrays.asList(racks));
LOG.info("newCapacity= " + newCapacity);
@@ -346,17 +480,75 @@ public class TestBalancer {
long totalUsedSpace = totalCapacity*3/10;
createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
(short) numOfDatanodes, 0);
- // start up an empty node with the same capacity and on the same rack
- cluster.startDataNodes(conf, 1, true, null,
- new String[]{newRack}, new long[]{newCapacity});
- totalCapacity += newCapacity;
+ if (nodes == null) { // there is no specification of new nodes.
+ // start up an empty node with the same capacity and on the same rack
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[]{newRack}, null,new long[]{newCapacity});
+ totalCapacity += newCapacity;
+ } else {
+ //if running a test with "include list", include original nodes as well
+ if (nodes.getNumberofIncludeNodes()>0) {
+ for (DataNode dn: cluster.getDataNodes())
+ nodes.getNodesToBeIncluded().add(dn.getDatanodeId().getHostName());
+ }
+ String[] newRacks = new String[nodes.getNumberofNewNodes()];
+ long[] newCapacities = new long[nodes.getNumberofNewNodes()];
+ for (int i=0; i < nodes.getNumberofNewNodes(); i++) {
+ newRacks[i] = newRack;
+ newCapacities[i] = newCapacity;
+ }
+ // if host names are specified for the new nodes to be created.
+ if (nodes.getNames() != null) {
+ cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null,
+ newRacks, nodes.getNames(), newCapacities);
+ totalCapacity += newCapacity*nodes.getNumberofNewNodes();
+ } else { // host names are not specified
+ cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null,
+ newRacks, null, newCapacities);
+ totalCapacity += newCapacity*nodes.getNumberofNewNodes();
+ //populate the include nodes
+ if (nodes.getNumberofIncludeNodes() > 0) {
+ int totalNodes = cluster.getDataNodes().size();
+ for (int i=0; i < nodes.getNumberofIncludeNodes(); i++) {
+ nodes.getNodesToBeIncluded().add (cluster.getDataNodes().get(
+ totalNodes-1-i).getDatanodeId().getXferAddr());
+ }
+ }
+ //polulate the exclude nodes
+ if (nodes.getNumberofExcludeNodes() > 0) {
+ int totalNodes = cluster.getDataNodes().size();
+ for (int i=0; i < nodes.getNumberofExcludeNodes(); i++) {
+ nodes.getNodesToBeExcluded().add (cluster.getDataNodes().get(
+ totalNodes-1-i).getDatanodeId().getXferAddr());
+ }
+ }
+ }
+ }
+ // run balancer and validate results
+ Balancer.Parameters p = Balancer.Parameters.DEFAULT;
+ if (nodes != null) {
+ p = new Balancer.Parameters(
+ Balancer.Parameters.DEFAULT.policy,
+ Balancer.Parameters.DEFAULT.threshold,
+ nodes.getNodesToBeExcluded(), nodes.getNodesToBeIncluded());
+ }
+
+ int expectedExcludedNodes = 0;
+ if (nodes != null) {
+ if (!nodes.getNodesToBeExcluded().isEmpty()) {
+ expectedExcludedNodes = nodes.getNodesToBeExcluded().size();
+ } else if (!nodes.getNodesToBeIncluded().isEmpty()) {
+ expectedExcludedNodes =
+ cluster.getDataNodes().size() - nodes.getNodesToBeIncluded().size();
+ }
+ }
// run balancer and validate results
if (useTool) {
- runBalancerCli(conf, totalUsedSpace, totalCapacity);
+ runBalancerCli(conf, totalUsedSpace, totalCapacity, p, useFile, expectedExcludedNodes);
} else {
- runBalancer(conf, totalUsedSpace, totalCapacity);
+ runBalancer(conf, totalUsedSpace, totalCapacity, p, expectedExcludedNodes);
}
} finally {
cluster.shutdown();
@@ -365,11 +557,17 @@ public class TestBalancer {
private void runBalancer(Configuration conf,
long totalUsedSpace, long totalCapacity) throws Exception {
+ runBalancer(conf, totalUsedSpace, totalCapacity, Balancer.Parameters.DEFAULT, 0);
+ }
+
+ private void runBalancer(Configuration conf,
+ long totalUsedSpace, long totalCapacity, Balancer.Parameters p,
+ int excludedNodes) throws Exception {
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
+ final int r = Balancer.run(namenodes, p, conf);
if (conf.getInt(DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT) ==0) {
assertEquals(Balancer.ReturnStatus.NO_MOVE_PROGRESS.code, r);
@@ -379,22 +577,66 @@ public class TestBalancer {
}
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
LOG.info("Rebalancing with default ctor.");
- waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, excludedNodes);
}
-
+
private void runBalancerCli(Configuration conf,
- long totalUsedSpace, long totalCapacity) throws Exception {
+ long totalUsedSpace, long totalCapacity,
+ Balancer.Parameters p, boolean useFile, int expectedExcludedNodes) throws Exception {
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
+ List <String> args = new ArrayList<String>();
+ args.add("-policy");
+ args.add("datanode");
+
+ File excludeHostsFile = null;
+ if (!p.nodesToBeExcluded.isEmpty()) {
+ args.add("-exclude");
+ if (useFile) {
+ excludeHostsFile = new File ("exclude-hosts-file");
+ PrintWriter pw = new PrintWriter(excludeHostsFile);
+ for (String host: p.nodesToBeExcluded) {
+ pw.write( host + "\n");
+ }
+ pw.close();
+ args.add("-f");
+ args.add("exclude-hosts-file");
+ } else {
+ args.add(StringUtils.join(p.nodesToBeExcluded, ','));
+ }
+ }
+
+ File includeHostsFile = null;
+ if (!p.nodesToBeIncluded.isEmpty()) {
+ args.add("-include");
+ if (useFile) {
+ includeHostsFile = new File ("include-hosts-file");
+ PrintWriter pw = new PrintWriter(includeHostsFile);
+ for (String host: p.nodesToBeIncluded){
+ pw.write( host + "\n");
+ }
+ pw.close();
+ args.add("-f");
+ args.add("include-hosts-file");
+ } else {
+ args.add(StringUtils.join(p.nodesToBeIncluded, ','));
+ }
+ }
- final String[] args = { "-policy", "datanode" };
final Tool tool = new Cli();
tool.setConf(conf);
- final int r = tool.run(args); // start rebalancing
+ final int r = tool.run(args.toArray(new String[0])); // start rebalancing
assertEquals("Tools should exit 0 on success", 0, r);
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
LOG.info("Rebalancing with default ctor.");
- waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, expectedExcludedNodes);
+
+ if (excludeHostsFile != null && excludeHostsFile.exists()) {
+ excludeHostsFile.delete();
+ }
+ if (includeHostsFile != null && includeHostsFile.exists()) {
+ includeHostsFile.delete();
+ }
}
/** one-node cluster test*/
@@ -440,7 +682,7 @@ public class TestBalancer {
}
}
- /** Test a cluster with even distribution,
+ /** Test a cluster with even distribution,
* then a new empty node is added to the cluster*/
@Test(timeout=100000)
public void testBalancer0() throws Exception {
@@ -554,7 +796,13 @@ public class TestBalancer {
} catch (IllegalArgumentException e) {
}
+ parameters = new String[] {"-include", "testnode1", "-exclude", "testnode2"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail("IllegalArgumentException is expected when both -exclude and -include are specified");
+ } catch (IllegalArgumentException e) {
+ }
}
/**
@@ -570,6 +818,183 @@ public class TestBalancer {
}
/**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithExcludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ excludeHosts, Parameters.DEFAULT.nodesToBeIncluded), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithExcludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"}, excludeHosts,
+ Parameters.DEFAULT.nodesToBeIncluded), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list in a file
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ excludeHosts, Parameters.DEFAULT.nodesToBeIncluded), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,G
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListWithPortsInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithIncludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithIncludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListWithPortsInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, true);
+ }
+
+ /**
* @param args
*/
public static void main(String[] args) throws Exception {
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java Fri Aug 1 01:29:49 2014
@@ -97,10 +97,10 @@ public class TestBalancerWithHANameNodes
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
assertEquals(1, namenodes.size());
assertTrue(namenodes.contains(HATestUtil.getLogicalUri(cluster)));
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
TestBalancer.waitForBalancer(totalUsedSpace, totalCapacity, client,
- cluster);
+ cluster, Balancer.Parameters.DEFAULT);
} finally {
cluster.shutdown();
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Fri Aug 1 01:29:49 2014
@@ -159,7 +159,7 @@ public class TestBalancerWithMultipleNam
// start rebalancing
final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(s.conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, s.conf);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, s.conf);
Assert.assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
LOG.info("BALANCER 2");
@@ -195,7 +195,7 @@ public class TestBalancerWithMultipleNam
balanced = true;
for(int d = 0; d < used.length; d++) {
final double p = used[d]*100.0/cap[d];
- balanced = p <= avg + Balancer.Parameters.DEFALUT.threshold;
+ balanced = p <= avg + Balancer.Parameters.DEFAULT.threshold;
if (!balanced) {
if (i % 100 == 0) {
LOG.warn("datanodes " + d + " is not yet balanced: "
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Fri Aug 1 01:29:49 2014
@@ -175,7 +175,7 @@ public class TestBalancerWithNodeGroup {
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
waitForHeartBeat(totalUsedSpace, totalCapacity);
@@ -189,7 +189,7 @@ public class TestBalancerWithNodeGroup {
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
Assert.assertTrue(r == Balancer.ReturnStatus.SUCCESS.code ||
(r == Balancer.ReturnStatus.NO_MOVE_PROGRESS.code));
waitForHeartBeat(totalUsedSpace, totalCapacity);
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Fri Aug 1 01:29:49 2014
@@ -590,7 +590,6 @@ public class TestBlockRecovery {
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, "1000");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(MiniDFSNNTopology.simpleSingleNN(8020, 50070))
.numDataNodes(1).build();
try {
cluster.waitClusterUp();
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java Fri Aug 1 01:29:49 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -1256,6 +1257,33 @@ public abstract class FSAclBaseTest {
fsAsDiana.getAclStatus(bruceFile);
}
+ @Test
+ public void testAccess() throws IOException, InterruptedException {
+ Path p1 = new Path("/p1");
+ fs.mkdirs(p1);
+ fs.setOwner(p1, BRUCE.getShortUserName(), "groupX");
+ fsAsBruce.setAcl(p1, Lists.newArrayList(
+ aclEntry(ACCESS, USER, READ),
+ aclEntry(ACCESS, USER, "bruce", READ),
+ aclEntry(ACCESS, GROUP, NONE),
+ aclEntry(ACCESS, OTHER, NONE)));
+ fsAsBruce.access(p1, FsAction.READ);
+ try {
+ fsAsBruce.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad/bad");
+ try {
+ fsAsBruce.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+
/**
* Creates a FileSystem for the super-user.
*
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Fri Aug 1 01:29:49 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSClient;
@@ -606,6 +607,7 @@ public class TestINodeFile {
fs.getAclStatus(testFileInodePath);
fs.getXAttrs(testFileInodePath);
fs.listXAttrs(testFileInodePath);
+ fs.access(testFileInodePath, FsAction.READ_WRITE);
}
// symbolic link related tests
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java Fri Aug 1 01:29:49 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -674,6 +675,13 @@ public class TestAclWithSnapshot {
} catch (AccessControlException e) {
// expected
}
+
+ try {
+ fs.access(pathToCheck, FsAction.READ);
+ fail("The access call should have failed for "+pathToCheck);
+ } catch (AccessControlException e) {
+ // expected
+ }
}
/**
@@ -689,6 +697,7 @@ public class TestAclWithSnapshot {
UserGroupInformation user, Path pathToCheck) throws Exception {
try {
fs.listStatus(pathToCheck);
+ fs.access(pathToCheck, FsAction.READ);
} catch (AccessControlException e) {
fail("expected permission granted for user " + user + ", path = " +
pathToCheck);
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Fri Aug 1 01:29:49 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -49,6 +50,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
+import org.junit.Test;
public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
private static final Configuration conf = new Configuration();
@@ -530,4 +532,35 @@ public class TestWebHdfsFileSystemContra
}
}
}
+
+ @Test
+ public void testAccess() throws IOException, InterruptedException {
+ Path p1 = new Path("/pathX");
+ try {
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting("alpha",
+ new String[]{"beta"});
+ WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, conf,
+ WebHdfsFileSystem.SCHEME);
+
+ fs.mkdirs(p1);
+ fs.setPermission(p1, new FsPermission((short) 0444));
+ fs.access(p1, FsAction.READ);
+ try {
+ fs.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad");
+ try {
+ fs.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ } finally {
+ fs.delete(p1, true);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java Fri Aug 1 01:29:49 2014
@@ -31,6 +31,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -283,6 +285,28 @@ public class TestWebHdfsUrl {
},
fileStatusUrl);
}
+
+ @Test(timeout=60000)
+ public void testCheckAccessUrl() throws IOException {
+ Configuration conf = new Configuration();
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("test-user");
+ UserGroupInformation.setLoginUser(ugi);
+
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
+ Path fsPath = new Path("/p1");
+
+ URL checkAccessUrl = webhdfs.toUrl(GetOpParam.Op.CHECKACCESS,
+ fsPath, new FsActionParam(FsAction.READ_WRITE));
+ checkQueryParams(
+ new String[]{
+ GetOpParam.Op.CHECKACCESS.toQueryString(),
+ new UserParam(ugi.getShortUserName()).toString(),
+ FsActionParam.NAME + "=" + FsAction.READ_WRITE.SYMBOL
+ },
+ checkAccessUrl);
+ }
private void checkQueryParams(String[] expected, URL url) {
Arrays.sort(expected);
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java Fri Aug 1 01:29:49 2014
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.FileNotFoundException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestWrapper;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -393,4 +395,37 @@ public class TestPermissionSymlinks {
GenericTestUtils.assertExceptionContains("Permission denied", e);
}
}
+
+ @Test
+ public void testAccess() throws Exception {
+ fs.setPermission(target, new FsPermission((short) 0002));
+ fs.setAcl(target, Arrays.asList(
+ aclEntry(ACCESS, USER, ALL),
+ aclEntry(ACCESS, GROUP, NONE),
+ aclEntry(ACCESS, USER, user.getShortUserName(), WRITE),
+ aclEntry(ACCESS, OTHER, WRITE)));
+ FileContext myfc = user.doAs(new PrivilegedExceptionAction<FileContext>() {
+ @Override
+ public FileContext run() throws IOException {
+ return FileContext.getFileContext(conf);
+ }
+ });
+
+ // Path to targetChild via symlink
+ myfc.access(link, FsAction.WRITE);
+ try {
+ myfc.access(link, FsAction.ALL);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path(link, "bad");
+ try {
+ myfc.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
}