You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/05/19 21:29:34 UTC
svn commit: r1596000 - in
/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs/
hadoop-hdfs/src/main/bin/ hadoo...
Author: wang
Date: Mon May 19 19:29:30 2014
New Revision: 1596000
URL: http://svn.apache.org/r1596000
Log:
Merge trunk r1595999 to branch.
Removed:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ScopedAclEntries.java
Modified:
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FsAclPermission.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1594887-1595999
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java Mon May 19 19:29:30 2014
@@ -32,14 +32,14 @@ import org.apache.hadoop.mount.MountdBas
*/
public class Mountd extends MountdBase {
- public Mountd(Configuration config, DatagramSocket registrationSocket)
- throws IOException {
- super(new RpcProgramMountd(config, registrationSocket));
+ public Mountd(Configuration config, DatagramSocket registrationSocket,
+ boolean allowInsecurePorts) throws IOException {
+ super(new RpcProgramMountd(config, registrationSocket, allowInsecurePorts));
}
public static void main(String[] args) throws IOException {
Configuration config = new Configuration();
- Mountd mountd = new Mountd(config, null);
+ Mountd mountd = new Mountd(config, null, true);
mountd.start(true);
}
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java Mon May 19 19:29:30 2014
@@ -79,11 +79,11 @@ public class RpcProgramMountd extends Rp
private final NfsExports hostsMatcher;
- public RpcProgramMountd(Configuration config,
- DatagramSocket registrationSocket) throws IOException {
+ public RpcProgramMountd(Configuration config, DatagramSocket registrationSocket,
+ boolean allowInsecurePorts) throws IOException {
// Note that RPC cache is not enabled
super("mountd", "localhost", config.getInt("nfs3.mountd.port", PORT),
- PROGRAM, VERSION_1, VERSION_3, registrationSocket);
+ PROGRAM, VERSION_1, VERSION_3, registrationSocket, allowInsecurePorts);
exports = new ArrayList<String>();
exports.add(config.get(Nfs3Constant.EXPORT_POINT,
Nfs3Constant.EXPORT_POINT_DEFAULT));
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java Mon May 19 19:29:30 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.net.DatagramSocket;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.nfs.mount.Mountd;
import org.apache.hadoop.nfs.nfs3.Nfs3Base;
import org.apache.hadoop.util.StringUtils;
@@ -41,12 +42,13 @@ public class Nfs3 extends Nfs3Base {
}
public Nfs3(Configuration conf) throws IOException {
- this(conf, null);
+ this(conf, null, true);
}
- public Nfs3(Configuration conf, DatagramSocket registrationSocket) throws IOException {
- super(new RpcProgramNfs3(conf, registrationSocket), conf);
- mountd = new Mountd(conf, registrationSocket);
+ public Nfs3(Configuration conf, DatagramSocket registrationSocket,
+ boolean allowInsecurePorts) throws IOException {
+ super(new RpcProgramNfs3(conf, registrationSocket, allowInsecurePorts), conf);
+ mountd = new Mountd(conf, registrationSocket, allowInsecurePorts);
}
public Mountd getMountd() {
@@ -61,8 +63,13 @@ public class Nfs3 extends Nfs3Base {
static void startService(String[] args,
DatagramSocket registrationSocket) throws IOException {
- StringUtils.startupShutdownMessage(Nfs3.class, args, LOG);
- final Nfs3 nfsServer = new Nfs3(new Configuration(), registrationSocket);
+ StringUtils.startupShutdownMessage(Nfs3.class, args, LOG);
+ Configuration conf = new Configuration();
+ boolean allowInsecurePorts = conf.getBoolean(
+ DFSConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_KEY,
+ DFSConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT);
+ final Nfs3 nfsServer = new Nfs3(new Configuration(), registrationSocket,
+ allowInsecurePorts);
nfsServer.startServiceInternal(true);
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java Mon May 19 19:29:30 2014
@@ -166,11 +166,12 @@ public class RpcProgramNfs3 extends RpcP
private final RpcCallCache rpcCallCache;
- public RpcProgramNfs3(Configuration config, DatagramSocket registrationSocket)
- throws IOException {
+ public RpcProgramNfs3(Configuration config, DatagramSocket registrationSocket,
+ boolean allowInsecurePorts) throws IOException {
super("NFS3", "localhost", config.getInt(Nfs3Constant.NFS3_SERVER_PORT,
Nfs3Constant.NFS3_SERVER_PORT_DEFAULT), Nfs3Constant.PROGRAM,
- Nfs3Constant.VERSION, Nfs3Constant.VERSION, registrationSocket);
+ Nfs3Constant.VERSION, Nfs3Constant.VERSION, registrationSocket,
+ allowInsecurePorts);
config.set(FsPermission.UMASK_LABEL, "000");
iug = new IdUserGroup();
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon May 19 19:29:30 2014
@@ -273,6 +273,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6334. Client failover proxy provider for IP failover based NN HA.
(kihwal)
+ HDFS-6406. Add capability for NFS gateway to reject connections from
+ unprivileged ports. (atm)
+
IMPROVEMENTS
HDFS-6007. Update documentation about short-circuit local reads (iwasakims
@@ -363,6 +366,12 @@ Release 2.5.0 - UNRELEASED
HDFS-6287. Add vecsum test of libhdfs read access times (cmccabe)
+ HDFS-5683. Better audit log messages for caching operations.
+ (Abhiraj Butala via wang)
+
+ HDFS-6345. DFS.listCacheDirectives() should allow filtering based on
+ cache directive ID. (wang)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -481,6 +490,14 @@ Release 2.5.0 - UNRELEASED
HDFS-6381. Fix a typo in INodeReference.java. (Binglin Chang via jing9)
+ HDFS-6400. Cannot execute hdfs oiv_legacy. (Akira AJISAKA via kihwal)
+
+ HDFS-6250. Fix test failed in TestBalancerWithNodeGroup.testBalancerWithRackLocality
+ (Binglin Chang and Chen He via junping_du)
+
+ HDFS-4913. Deleting file through fuse-dfs when using trash fails requiring
+ root permissions (cmccabe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -549,6 +566,21 @@ Release 2.4.1 - UNRELEASED
HDFS-6326. WebHdfs ACL compatibility is broken. (cnauroth)
+ HDFS-6361. TestIdUserGroup.testUserUpdateSetting failed due to out of range
+ nfsnobody Id. (Yongjun Zhang via brandonli)
+
+ HDFS-6362. InvalidateBlocks is inconsistent in usage of DatanodeUuid and
+ StorageID. (Arpit Agarwal)
+
+ HDFS-6402. Suppress findbugs warning for failure to override equals and
+ hashCode in FsAclPermission. (cnauroth)
+
+ HDFS-6325. Append should fail if the last block has insufficient number of
+ replicas (Keith Pak via cos)
+
+ HDFS-6397. NN shows inconsistent value in deadnode count.
+ (Mohammad Kamrul Islam via kihwal)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Mon May 19 19:29:30 2014
@@ -162,7 +162,7 @@ elif [ "$COMMAND" = "jmxget" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
elif [ "$COMMAND" = "oiv" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
-elif [ "COMMAND" = "oiv_legacy" ] ; then
+elif [ "$COMMAND" = "oiv_legacy" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
elif [ "$COMMAND" = "oev" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1594887-1595999
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon May 19 19:29:30 2014
@@ -638,9 +638,12 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE =
"dfs.client.hedged.read.threadpool.size";
- public static final int DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE = 0;
- public static final String DFS_NFS_KEYTAB_FILE_KEY = "dfs.nfs.keytab.file";
- public static final String DFS_NFS_KERBEROS_PRINCIPAL_KEY = "dfs.nfs.kerberos.principal";
- public static final String DFS_NFS_REGISTRATION_PORT_KEY = "dfs.nfs.registration.port";
- public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
+ public static final int DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE = 0;
+ public static final String DFS_NFS_KEYTAB_FILE_KEY = "dfs.nfs.keytab.file";
+ public static final String DFS_NFS_KERBEROS_PRINCIPAL_KEY = "dfs.nfs.kerberos.principal";
+ public static final String DFS_NFS_REGISTRATION_PORT_KEY = "dfs.nfs.registration.port";
+ public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
+ public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "dfs.nfs.allow.insecure.ports";
+ public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true;
+
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java Mon May 19 19:29:30 2014
@@ -23,6 +23,10 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.ipc.RemoteException;
+
+import com.google.common.base.Preconditions;
/**
* CacheDirectiveIterator is a remote iterator that iterates cache directives.
@@ -33,7 +37,7 @@ import org.apache.hadoop.fs.BatchedRemot
public class CacheDirectiveIterator
extends BatchedRemoteIterator<Long, CacheDirectiveEntry> {
- private final CacheDirectiveInfo filter;
+ private CacheDirectiveInfo filter;
private final ClientProtocol namenode;
public CacheDirectiveIterator(ClientProtocol namenode,
@@ -43,10 +47,72 @@ public class CacheDirectiveIterator
this.filter = filter;
}
+ private static CacheDirectiveInfo removeIdFromFilter(CacheDirectiveInfo filter) {
+ CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(filter);
+ builder.setId(null);
+ return builder.build();
+ }
+
+ /**
+ * Used for compatibility when communicating with a server version that
+ * does not support filtering directives by ID.
+ */
+ private static class SingleEntry implements
+ BatchedEntries<CacheDirectiveEntry> {
+
+ private final CacheDirectiveEntry entry;
+
+ public SingleEntry(final CacheDirectiveEntry entry) {
+ this.entry = entry;
+ }
+
+ @Override
+ public CacheDirectiveEntry get(int i) {
+ if (i > 0) {
+ return null;
+ }
+ return entry;
+ }
+
+ @Override
+ public int size() {
+ return 1;
+ }
+
+ @Override
+ public boolean hasMore() {
+ return false;
+ }
+ }
+
@Override
public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
throws IOException {
- return namenode.listCacheDirectives(prevKey, filter);
+ BatchedEntries<CacheDirectiveEntry> entries = null;
+ try {
+ entries = namenode.listCacheDirectives(prevKey, filter);
+ } catch (IOException e) {
+ if (e.getMessage().contains("Filtering by ID is unsupported")) {
+ // Retry case for old servers, do the filtering client-side
+ long id = filter.getId();
+ filter = removeIdFromFilter(filter);
+ // Using id - 1 as prevId should get us a window containing the id
+ // This is somewhat brittle, since it depends on directives being
+ // returned in order of ascending ID.
+ entries = namenode.listCacheDirectives(id - 1, filter);
+ for (int i=0; i<entries.size(); i++) {
+ CacheDirectiveEntry entry = entries.get(i);
+ if (entry.getInfo().getId().equals((Long)id)) {
+ return new SingleEntry(entry);
+ }
+ }
+ throw new RemoteException(InvalidRequestException.class.getName(),
+ "Did not find requested id " + id);
+ }
+ throw e;
+ }
+ Preconditions.checkNotNull(entries);
+ return entries;
}
@Override
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FsAclPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FsAclPermission.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FsAclPermission.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FsAclPermission.java Mon May 19 19:29:30 2014
@@ -60,4 +60,18 @@ public class FsAclPermission extends FsP
public boolean getAclBit() {
return aclBit;
}
+
+ @Override
+ public boolean equals(Object o) {
+ // This intentionally delegates to the base class. This is only overridden
+ // to suppress a FindBugs warning.
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // This intentionally delegates to the base class. This is only overridden
+ // to suppress a FindBugs warning.
+ return super.hashCode();
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java Mon May 19 19:29:30 2014
@@ -170,7 +170,7 @@ class NameNodeConnector {
}
/* The idea for making sure that there is no more than one balancer
- * running in an HDFS is to create a file in the HDFS, writes the IP address
+ * running in an HDFS is to create a file in the HDFS, writes the hostname
* of the machine on which the balancer is running to the file, but did not
* close the file until the balancer exits.
* This prevents the second balancer from running because it can not
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Mon May 19 19:29:30 2014
@@ -265,7 +265,8 @@ public class BlockManager {
final long pendingPeriod = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY,
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_DEFAULT);
- invalidateBlocks = new InvalidateBlocks(datanodeManager, pendingPeriod);
+ invalidateBlocks = new InvalidateBlocks(
+ datanodeManager.blockInvalidateLimit, pendingPeriod);
// Compute the map capacity by allocating 2% of total memory
blocksMap = new BlocksMap(
@@ -701,7 +702,7 @@ public class BlockManager {
// remove this block from the list of pending blocks to be deleted.
for (DatanodeStorageInfo storage : targets) {
- invalidateBlocks.remove(storage.getStorageID(), oldBlock);
+ invalidateBlocks.remove(storage.getDatanodeDescriptor(), oldBlock);
}
// Adjust safe-mode totals, since under-construction blocks don't
@@ -726,7 +727,7 @@ public class BlockManager {
for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
final String storageID = storage.getStorageID();
// filter invalidate replicas
- if(!invalidateBlocks.contains(storageID, block)) {
+ if(!invalidateBlocks.contains(storage.getDatanodeDescriptor(), block)) {
locations.add(storage);
}
}
@@ -945,6 +946,16 @@ public class BlockManager {
}
/**
+ * Check if a block is replicated to at least the minimum replication.
+ */
+ public boolean isSufficientlyReplicated(BlockInfo b) {
+ // Compare against the lesser of the minReplication and number of live DNs.
+ final int replication =
+ Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes());
+ return countNodes(b).liveReplicas() >= replication;
+ }
+
+ /**
* return a list of blocks & their locations on <code>datanode</code> whose
* total size is <code>size</code>
*
@@ -1016,7 +1027,7 @@ public class BlockManager {
pendingDNMessages.removeAllMessagesForDatanode(node);
node.resetBlocks();
- invalidateBlocks.remove(node.getDatanodeUuid());
+ invalidateBlocks.remove(node);
// If the DN hasn't block-reported since the most recent
// failover, then we may have been holding up on processing
@@ -1184,7 +1195,7 @@ public class BlockManager {
* @return total number of block for deletion
*/
int computeInvalidateWork(int nodesToProcess) {
- final List<String> nodes = invalidateBlocks.getStorageIDs();
+ final List<DatanodeInfo> nodes = invalidateBlocks.getDatanodes();
Collections.shuffle(nodes);
nodesToProcess = Math.min(nodes.size(), nodesToProcess);
@@ -1973,7 +1984,7 @@ public class BlockManager {
}
// Ignore replicas already scheduled to be removed from the DN
- if(invalidateBlocks.contains(dn.getDatanodeUuid(), block)) {
+ if(invalidateBlocks.contains(dn, block)) {
/*
* TODO: following assertion is incorrect, see HDFS-2668 assert
* storedBlock.findDatanode(dn) < 0 : "Block " + block +
@@ -3199,9 +3210,8 @@ public class BlockManager {
*
* @return number of blocks scheduled for removal during this iteration.
*/
- private int invalidateWorkForOneNode(String nodeId) {
+ private int invalidateWorkForOneNode(DatanodeInfo dn) {
final List<Block> toInvalidate;
- final DatanodeDescriptor dn;
namesystem.writeLock();
try {
@@ -3210,15 +3220,13 @@ public class BlockManager {
LOG.debug("In safemode, not computing replication work");
return 0;
}
- // get blocks to invalidate for the nodeId
- assert nodeId != null;
- dn = datanodeManager.getDatanode(nodeId);
- if (dn == null) {
- invalidateBlocks.remove(nodeId);
- return 0;
- }
- toInvalidate = invalidateBlocks.invalidateWork(nodeId, dn);
- if (toInvalidate == null) {
+ try {
+ toInvalidate = invalidateBlocks.invalidateWork(datanodeManager.getDatanode(dn));
+
+ if (toInvalidate == null) {
+ return 0;
+ }
+ } catch(UnregisteredNodeException une) {
return 0;
}
} finally {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Mon May 19 19:29:30 2014
@@ -1057,15 +1057,7 @@ public class DatanodeManager {
/** @return the number of dead datanodes. */
public int getNumDeadDataNodes() {
- int numDead = 0;
- synchronized (datanodeMap) {
- for(DatanodeDescriptor dn : datanodeMap.values()) {
- if (isDatanodeDead(dn) ) {
- numDead++;
- }
- }
- }
- return numDead;
+ return getDatanodeListForReport(DatanodeReportType.DEAD).size();
}
/** @return list of datanodes where decommissioning is in progress. */
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java Mon May 19 19:29:30 2014
@@ -44,13 +44,13 @@ import com.google.common.annotations.Vis
*/
@InterfaceAudience.Private
class InvalidateBlocks {
- /** Mapping: StorageID -> Collection of Blocks */
- private final Map<String, LightWeightHashSet<Block>> node2blocks =
- new TreeMap<String, LightWeightHashSet<Block>>();
+ /** Mapping: DatanodeInfo -> Collection of Blocks */
+ private final Map<DatanodeInfo, LightWeightHashSet<Block>> node2blocks =
+ new TreeMap<DatanodeInfo, LightWeightHashSet<Block>>();
/** The total number of blocks in the map. */
private long numBlocks = 0L;
- private final DatanodeManager datanodeManager;
+ private final int blockInvalidateLimit;
/**
* The period of pending time for block invalidation since the NameNode
@@ -60,8 +60,8 @@ class InvalidateBlocks {
/** the startup time */
private final long startupTime = Time.monotonicNow();
- InvalidateBlocks(final DatanodeManager datanodeManager, long pendingPeriodInMs) {
- this.datanodeManager = datanodeManager;
+ InvalidateBlocks(final int blockInvalidateLimit, long pendingPeriodInMs) {
+ this.blockInvalidateLimit = blockInvalidateLimit;
this.pendingPeriodInMs = pendingPeriodInMs;
printBlockDeletionTime(BlockManager.LOG);
}
@@ -86,12 +86,9 @@ class InvalidateBlocks {
* invalidation. Blocks are compared including their generation stamps:
* if a block is pending invalidation but with a different generation stamp,
* returns false.
- * @param storageID the storage to check
- * @param the block to look for
- *
*/
- synchronized boolean contains(final String storageID, final Block block) {
- final LightWeightHashSet<Block> s = node2blocks.get(storageID);
+ synchronized boolean contains(final DatanodeInfo dn, final Block block) {
+ final LightWeightHashSet<Block> s = node2blocks.get(dn);
if (s == null) {
return false; // no invalidate blocks for this storage ID
}
@@ -106,10 +103,10 @@ class InvalidateBlocks {
*/
synchronized void add(final Block block, final DatanodeInfo datanode,
final boolean log) {
- LightWeightHashSet<Block> set = node2blocks.get(datanode.getDatanodeUuid());
+ LightWeightHashSet<Block> set = node2blocks.get(datanode);
if (set == null) {
set = new LightWeightHashSet<Block>();
- node2blocks.put(datanode.getDatanodeUuid(), set);
+ node2blocks.put(datanode, set);
}
if (set.add(block)) {
numBlocks++;
@@ -121,20 +118,20 @@ class InvalidateBlocks {
}
/** Remove a storage from the invalidatesSet */
- synchronized void remove(final String storageID) {
- final LightWeightHashSet<Block> blocks = node2blocks.remove(storageID);
+ synchronized void remove(final DatanodeInfo dn) {
+ final LightWeightHashSet<Block> blocks = node2blocks.remove(dn);
if (blocks != null) {
numBlocks -= blocks.size();
}
}
/** Remove the block from the specified storage. */
- synchronized void remove(final String storageID, final Block block) {
- final LightWeightHashSet<Block> v = node2blocks.get(storageID);
+ synchronized void remove(final DatanodeInfo dn, final Block block) {
+ final LightWeightHashSet<Block> v = node2blocks.get(dn);
if (v != null && v.remove(block)) {
numBlocks--;
if (v.isEmpty()) {
- node2blocks.remove(storageID);
+ node2blocks.remove(dn);
}
}
}
@@ -148,18 +145,18 @@ class InvalidateBlocks {
return;
}
- for(Map.Entry<String,LightWeightHashSet<Block>> entry : node2blocks.entrySet()) {
+ for(Map.Entry<DatanodeInfo, LightWeightHashSet<Block>> entry : node2blocks.entrySet()) {
final LightWeightHashSet<Block> blocks = entry.getValue();
if (blocks.size() > 0) {
- out.println(datanodeManager.getDatanode(entry.getKey()));
+ out.println(entry.getKey());
out.println(blocks);
}
}
}
/** @return a list of the storage IDs. */
- synchronized List<String> getStorageIDs() {
- return new ArrayList<String>(node2blocks.keySet());
+ synchronized List<DatanodeInfo> getDatanodes() {
+ return new ArrayList<DatanodeInfo>(node2blocks.keySet());
}
/**
@@ -170,8 +167,7 @@ class InvalidateBlocks {
return pendingPeriodInMs - (Time.monotonicNow() - startupTime);
}
- synchronized List<Block> invalidateWork(
- final String storageId, final DatanodeDescriptor dn) {
+ synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) {
final long delay = getInvalidationDelay();
if (delay > 0) {
if (BlockManager.LOG.isDebugEnabled()) {
@@ -181,18 +177,18 @@ class InvalidateBlocks {
}
return null;
}
- final LightWeightHashSet<Block> set = node2blocks.get(storageId);
+ final LightWeightHashSet<Block> set = node2blocks.get(dn);
if (set == null) {
return null;
}
// # blocks that can be sent in one message is limited
- final int limit = datanodeManager.blockInvalidateLimit;
+ final int limit = blockInvalidateLimit;
final List<Block> toInvalidate = set.pollN(limit);
// If we send everything in this message, remove this node entry
if (set.isEmpty()) {
- remove(storageId);
+ remove(dn);
}
dn.addBlocksToBeInvalidated(toInvalidate);
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java Mon May 19 19:29:30 2014
@@ -27,8 +27,10 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ScopedAclEntries;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -90,7 +92,7 @@ final class AclStorage {
FsPermission childPerm = child.getFsPermission();
// Copy each default ACL entry from parent to new child's access ACL.
- boolean parentDefaultIsMinimal = isMinimalAcl(parentDefaultEntries);
+ boolean parentDefaultIsMinimal = AclUtil.isMinimalAcl(parentDefaultEntries);
for (AclEntry entry: parentDefaultEntries) {
AclEntryType type = entry.getType();
String name = entry.getName();
@@ -127,7 +129,7 @@ final class AclStorage {
Collections.<AclEntry>emptyList();
final FsPermission newPerm;
- if (!isMinimalAcl(accessEntries) || !defaultEntries.isEmpty()) {
+ if (!AclUtil.isMinimalAcl(accessEntries) || !defaultEntries.isEmpty()) {
// Save the new ACL to the child.
child.addAclFeature(createAclFeature(accessEntries, defaultEntries));
newPerm = createFsPermissionForExtendedAcl(accessEntries, childPerm);
@@ -172,7 +174,7 @@ final class AclStorage {
FsPermission perm = inode.getFsPermission();
AclFeature f = inode.getAclFeature();
if (f == null) {
- return getMinimalAcl(perm);
+ return AclUtil.getMinimalAcl(perm);
}
final List<AclEntry> existingAcl;
@@ -208,7 +210,7 @@ final class AclStorage {
} else {
// It's possible that there is a default ACL but no access ACL. In this
// case, add the minimal access ACL implied by the permission bits.
- existingAcl.addAll(getMinimalAcl(perm));
+ existingAcl.addAll(AclUtil.getMinimalAcl(perm));
}
// Add all default entries after the access entries.
@@ -267,7 +269,7 @@ final class AclStorage {
assert newAcl.size() >= 3;
FsPermission perm = inode.getFsPermission();
final FsPermission newPerm;
- if (!isMinimalAcl(newAcl)) {
+ if (!AclUtil.isMinimalAcl(newAcl)) {
// This is an extended ACL. Split entries into access vs. default.
ScopedAclEntries scoped = new ScopedAclEntries(newAcl);
List<AclEntry> accessEntries = scoped.getAccessEntries();
@@ -321,7 +323,7 @@ final class AclStorage {
// For the access ACL, the feature only needs to hold the named user and
// group entries. For a correctly sorted ACL, these will be in a
// predictable range.
- if (!isMinimalAcl(accessEntries)) {
+ if (!AclUtil.isMinimalAcl(accessEntries)) {
featureEntries.addAll(
accessEntries.subList(1, accessEntries.size() - 2));
}
@@ -366,41 +368,4 @@ final class AclStorage {
accessEntries.get(2).getPermission(),
existingPerm.getStickyBit());
}
-
- /**
- * Translates the given permission bits to the equivalent minimal ACL.
- *
- * @param perm FsPermission to translate
- * @return List<AclEntry> containing exactly 3 entries representing the owner,
- * group and other permissions
- */
- private static List<AclEntry> getMinimalAcl(FsPermission perm) {
- return Lists.newArrayList(
- new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.USER)
- .setPermission(perm.getUserAction())
- .build(),
- new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.GROUP)
- .setPermission(perm.getGroupAction())
- .build(),
- new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS)
- .setType(AclEntryType.OTHER)
- .setPermission(perm.getOtherAction())
- .build());
- }
-
- /**
- * Checks if the given entries represent a minimal ACL (contains exactly 3
- * entries).
- *
- * @param entries List<AclEntry> entries to check
- * @return boolean true if the entries represent a minimal ACL
- */
- private static boolean isMinimalAcl(List<AclEntry> entries) {
- return entries.size() == 3;
- }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java Mon May 19 19:29:30 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ScopedAclEntries;
import org.apache.hadoop.hdfs.protocol.AclException;
/**
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java Mon May 19 19:29:30 2014
@@ -691,15 +691,25 @@ public final class CacheManager {
assert namesystem.hasReadLock();
final int NUM_PRE_ALLOCATED_ENTRIES = 16;
String filterPath = null;
- if (filter.getId() != null) {
- throw new IOException("Filtering by ID is unsupported.");
- }
if (filter.getPath() != null) {
filterPath = validatePath(filter);
}
if (filter.getReplication() != null) {
- throw new IOException("Filtering by replication is unsupported.");
+ throw new InvalidRequestException(
+ "Filtering by replication is unsupported.");
+ }
+
+ // Querying for a single ID
+ final Long id = filter.getId();
+ if (id != null) {
+ if (!directivesById.containsKey(id)) {
+ throw new InvalidRequestException("Did not find requested id " + id);
+ }
+ // Since we use a tailMap on directivesById, setting prev to id-1 gets
+ // us the directive with the id (if present)
+ prevId = id - 1;
}
+
ArrayList<CacheDirectiveEntry> replies =
new ArrayList<CacheDirectiveEntry>(NUM_PRE_ALLOCATED_ENTRIES);
int numReplies = 0;
@@ -711,6 +721,14 @@ public final class CacheManager {
}
CacheDirective curDirective = cur.getValue();
CacheDirectiveInfo info = cur.getValue().toInfo();
+
+ // If the requested ID is present, it should be the first item.
+ // Hitting this case means the ID is not present, or we're on the second
+ // item and should break out.
+ if (id != null &&
+ !(info.getId().equals(id))) {
+ break;
+ }
if (filter.getPool() != null &&
!info.getPool().equals(filter.getPool())) {
continue;
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon May 19 19:29:30 2014
@@ -2380,7 +2380,13 @@ public class FSNamesystem implements Nam
// finalizeINodeFileUnderConstruction so we need to refresh
// the referenced file.
myFile = INodeFile.valueOf(dir.getINode(src), src, true);
-
+ final BlockInfo lastBlock = myFile.getLastBlock();
+ // Check that the block has at least minimum replication.
+ if(lastBlock != null && lastBlock.isComplete() &&
+ !getBlockManager().isSufficientlyReplicated(lastBlock)) {
+ throw new IOException("append: lastBlock=" + lastBlock +
+ " of src=" + src + " is not sufficiently replicated yet.");
+ }
final DatanodeDescriptor clientNode =
blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
return prepareFileForWrite(src, myFile, holder, clientMachine, clientNode,
@@ -7443,6 +7449,7 @@ public class FSNamesystem implements Nam
cacheManager.waitForRescanIfNeeded();
}
writeLock();
+ String effectiveDirectiveStr = null;
Long result = null;
try {
checkOperation(OperationCategory.WRITE);
@@ -7454,11 +7461,12 @@ public class FSNamesystem implements Nam
throw new IOException("addDirective: you cannot specify an ID " +
"for this operation.");
}
- CacheDirectiveInfo effectiveDirective =
+ CacheDirectiveInfo effectiveDirective =
cacheManager.addDirective(directive, pc, flags);
getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
cacheEntry != null);
result = effectiveDirective.getId();
+ effectiveDirectiveStr = effectiveDirective.toString();
success = true;
} finally {
writeUnlock();
@@ -7466,7 +7474,7 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCacheDirective", null, null, null);
+ logAuditEvent(success, "addCacheDirective", effectiveDirectiveStr, null, null);
}
RetryCache.setState(cacheEntry, success, result);
}
@@ -7503,7 +7511,8 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "modifyCacheDirective", null, null, null);
+ String idStr = "{id: " + directive.getId().toString() + "}";
+ logAuditEvent(success, "modifyCacheDirective", idStr, directive.toString(), null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7531,7 +7540,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "removeCacheDirective", null, null,
+ String idStr = "{id: " + id.toString() + "}";
+ logAuditEvent(success, "removeCacheDirective", idStr, null,
null);
}
RetryCache.setState(cacheEntry, success);
@@ -7556,7 +7566,7 @@ public class FSNamesystem implements Nam
} finally {
readUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "listCacheDirectives", null, null,
+ logAuditEvent(success, "listCacheDirectives", filter.toString(), null,
null);
}
}
@@ -7573,6 +7583,7 @@ public class FSNamesystem implements Nam
}
writeLock();
boolean success = false;
+ String poolInfoStr = null;
try {
checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
@@ -7583,12 +7594,13 @@ public class FSNamesystem implements Nam
pc.checkSuperuserPrivilege();
}
CachePoolInfo info = cacheManager.addCachePool(req);
+ poolInfoStr = info.toString();
getEditLog().logAddCachePool(info, cacheEntry != null);
success = true;
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCachePool", req.getPoolName(), null, null);
+ logAuditEvent(success, "addCachePool", poolInfoStr, null, null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7621,7 +7633,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "modifyCachePool", req.getPoolName(), null, null);
+ String poolNameStr = "{poolName: " + req.getPoolName() + "}";
+ logAuditEvent(success, "modifyCachePool", poolNameStr, req.toString(), null);
}
RetryCache.setState(cacheEntry, success);
}
@@ -7654,7 +7667,8 @@ public class FSNamesystem implements Nam
} finally {
writeUnlock();
if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "removeCachePool", cachePoolName, null, null);
+ String poolNameStr = "{poolName: " + cachePoolName + "}";
+ logAuditEvent(success, "removeCachePool", poolNameStr, null, null);
}
RetryCache.setState(cacheEntry, success);
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Mon May 19 19:29:30 2014
@@ -503,19 +503,21 @@ public class CacheAdmin extends Configur
@Override
public String getShortUsage() {
- return "[" + getName() + " [-stats] [-path <path>] [-pool <pool>]]\n";
+ return "[" + getName()
+ + " [-stats] [-path <path>] [-pool <pool>] [-id <id>]\n";
}
@Override
public String getLongUsage() {
TableListing listing = getOptionDescriptionListing();
+ listing.addRow("-stats", "List path-based cache directive statistics.");
listing.addRow("<path>", "List only " +
"cache directives with this path. " +
"Note that if there is a cache directive for <path> " +
"in a cache pool that we don't have read access for, it " +
"will not be listed.");
listing.addRow("<pool>", "List only path cache directives in that pool.");
- listing.addRow("-stats", "List path-based cache directive statistics.");
+ listing.addRow("<id>", "List the cache directive with this id.");
return getShortUsage() + "\n" +
"List cache directives.\n\n" +
listing.toString();
@@ -534,6 +536,10 @@ public class CacheAdmin extends Configur
builder.setPool(poolFilter);
}
boolean printStats = StringUtils.popOption("-stats", args);
+ String idFilter = StringUtils.popOptionWithArgument("-id", args);
+ if (idFilter != null) {
+ builder.setId(Long.parseLong(idFilter));
+ }
if (!args.isEmpty()) {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java Mon May 19 19:29:30 2014
@@ -47,7 +47,7 @@ public class OfflineImageViewer {
public static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
private final static String usage =
- "Usage: bin/hdfs oiv [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
+ "Usage: bin/hdfs oiv_legacy [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
"Offline Image Viewer\n" +
"View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" +
"saving the results in OUTPUTFILE.\n" +
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1594887-1595999
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c Mon May 19 19:29:30 2014
@@ -16,111 +16,228 @@
* limitations under the License.
*/
-
#include <hdfs.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
#include <strings.h>
+#include "fuse_context_handle.h"
#include "fuse_dfs.h"
#include "fuse_trash.h"
-#include "fuse_context_handle.h"
-
-
-const char *const TrashPrefixDir = "/user/root/.Trash";
-const char *const TrashDir = "/user/root/.Trash/Current";
+#include "fuse_users.h"
#define TRASH_RENAME_TRIES 100
+#define ALREADY_IN_TRASH_ERR 9000
-//
-// NOTE: this function is a c implementation of org.apache.hadoop.fs.Trash.moveToTrash(Path path).
-//
-
-int move_to_trash(const char *item, hdfsFS userFS) {
-
- // retrieve dfs specific data
- dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
-
- // check params and the context var
- assert(item);
- assert(dfs);
- assert('/' == *item);
- assert(rindex(item,'/') >= 0);
-
-
- char fname[4096]; // or last element of the directory path
- char parent_dir[4096]; // the directory the fname resides in
-
- if (strlen(item) > sizeof(fname) - strlen(TrashDir)) {
- ERROR("Buffer too small to accomodate path of len %d", (int)strlen(item));
- return -EIO;
+/**
+ * Split a path into a parent directory and a base path component.
+ *
+ * @param abs_path The absolute path.
+ * @param pcomp (out param) Will be set to the last path component.
+ * Malloced.
+ * @param parent_dir (out param) Will be set to the parent directory.
+ * Malloced.
+ *
+ * @return 0 on success.
+ * On success, both *pcomp and *parent_dir will contain
+ * malloc'ed strings.
+ * EINVAL if the path wasn't absolute.
+ * EINVAL if there is no parent directory (i.e. abs_path=/)
+ * ENOMEM if we ran out of memory.
+ */
+static int get_parent_dir(const char *abs_path, char **pcomp,
+ char **parent_dir)
+{
+ int ret;
+ char *pdir = NULL, *pc = NULL, *last_slash;
+
+ pdir = strdup(abs_path);
+ if (!pdir) {
+ ret = ENOMEM;
+ goto done;
+ }
+ last_slash = rindex(pdir, '/');
+ if (!last_slash) {
+ ERROR("get_parent_dir(%s): expected absolute path.\n", abs_path);
+ ret = EINVAL;
+ goto done;
+ }
+ if (last_slash[1] == '\0') {
+ *last_slash = '\0';
+ last_slash = rindex(pdir, '/');
+ if (!last_slash) {
+ ERROR("get_parent_dir(%s): there is no parent dir.\n", abs_path);
+ ret = EINVAL;
+ goto done;
+ }
}
-
- // separate the file name and the parent directory of the item to be deleted
- {
- int length_of_parent_dir = rindex(item, '/') - item ;
- int length_of_fname = strlen(item) - length_of_parent_dir - 1; // the '/'
-
- // note - the below strncpys should be safe from overflow because of the check on item's string length above.
- strncpy(parent_dir, item, length_of_parent_dir);
- parent_dir[length_of_parent_dir ] = 0;
- strncpy(fname, item + length_of_parent_dir + 1, strlen(item));
- fname[length_of_fname + 1] = 0;
+ pc = strdup(last_slash + 1);
+ if (!pc) {
+ ret = ENOMEM;
+ goto done;
+ }
+ *last_slash = '\0';
+ ret = 0;
+done:
+ if (ret) {
+ free(pdir);
+ free(pc);
+ return ret;
}
+ *pcomp = pc;
+ *parent_dir = pdir;
+ return 0;
+}
- // create the target trash directory
- char trash_dir[4096];
- if (snprintf(trash_dir, sizeof(trash_dir), "%s%s", TrashDir, parent_dir)
- >= sizeof trash_dir) {
- ERROR("Move to trash error target not big enough for %s", item);
- return -EIO;
+/**
+ * Get the base path to the trash. This will depend on the user ID.
+ * For example, a user whose ID maps to 'foo' will get back the path
+ * "/user/foo/.Trash/Current".
+ *
+ * @param trash_base (out param) the base path to the trash.
+ * Malloced.
+ *
+ * @return 0 on success; error code otherwise.
+ */
+static int get_trash_base(char **trash_base)
+{
+ const char * const PREFIX = "/user/";
+ const char * const SUFFIX = "/.Trash/Current";
+ char *user_name = NULL, *base = NULL;
+ uid_t uid = fuse_get_context()->uid;
+ int ret;
+
+ user_name = getUsername(uid);
+ if (!user_name) {
+ ERROR("get_trash_base(): failed to get username for uid %"PRId64"\n",
+ (uint64_t)uid);
+ ret = EIO;
+ goto done;
+ }
+ if (asprintf(&base, "%s%s%s", PREFIX, user_name, SUFFIX) < 0) {
+ base = NULL;
+ ret = ENOMEM;
+ goto done;
+ }
+ ret = 0;
+done:
+ free(user_name);
+ if (ret) {
+ free(base);
+ return ret;
}
+ *trash_base = base;
+ return 0;
+}
+//
+// NOTE: this function is a c implementation of org.apache.hadoop.fs.Trash.moveToTrash(Path path).
+//
+int move_to_trash(const char *abs_path, hdfsFS userFS)
+{
+ int ret;
+ char *pcomp = NULL, *parent_dir = NULL, *trash_base = NULL;
+ char *target_dir = NULL, *target = NULL;
+
+ ret = get_parent_dir(abs_path, &pcomp, &parent_dir);
+ if (ret) {
+ goto done;
+ }
+ ret = get_trash_base(&trash_base);
+ if (ret) {
+ goto done;
+ }
+ if (!strncmp(trash_base, abs_path, strlen(trash_base))) {
+ INFO("move_to_trash(%s): file is already in the trash; deleting.",
+ abs_path);
+ ret = ALREADY_IN_TRASH_ERR;
+ goto done;
+ }
+ fprintf(stderr, "trash_base='%s'\n", trash_base);
+ if (asprintf(&target_dir, "%s%s", trash_base, parent_dir) < 0) {
+ ret = ENOMEM;
+ target_dir = NULL;
+ goto done;
+ }
+ if (asprintf(&target, "%s/%s", target_dir, pcomp) < 0) {
+ ret = ENOMEM;
+ target = NULL;
+ goto done;
+ }
// create the target trash directory in trash (if needed)
- if ( hdfsExists(userFS, trash_dir)) {
+ if (hdfsExists(userFS, target_dir) != 0) {
// make the directory to put it in in the Trash - NOTE
// hdfsCreateDirectory also creates parents, so Current will be created if it does not exist.
- if (hdfsCreateDirectory(userFS, trash_dir)) {
- return -EIO;
+ if (hdfsCreateDirectory(userFS, target_dir)) {
+ ret = errno;
+ ERROR("move_to_trash(%s) error: hdfsCreateDirectory(%s) failed with error %d",
+ abs_path, target_dir, ret);
+ goto done;
}
- }
-
- //
- // if the target path in Trash already exists, then append with
- // a number. Start from 1.
- //
- char target[4096];
- int j ;
- if ( snprintf(target, sizeof target,"%s/%s",trash_dir, fname) >= sizeof target) {
- ERROR("Move to trash error target not big enough for %s", item);
- return -EIO;
- }
-
- // NOTE: this loop differs from the java version by capping the #of tries
- for (j = 1; ! hdfsExists(userFS, target) && j < TRASH_RENAME_TRIES ; j++) {
- if (snprintf(target, sizeof target,"%s/%s.%d",trash_dir, fname, j) >= sizeof target) {
- ERROR("Move to trash error target not big enough for %s", item);
- return -EIO;
+ } else if (hdfsExists(userFS, target) == 0) {
+ // If there is already a file in the trash with this path, append a number.
+ int idx;
+ for (idx = 1; idx < TRASH_RENAME_TRIES; idx++) {
+ free(target);
+ if (asprintf(&target, "%s%s.%d", target_dir, pcomp, idx) < 0) {
+ target = NULL;
+ ret = ENOMEM;
+ goto done;
+ }
+ if (hdfsExists(userFS, target) != 0) {
+ break;
+ }
+ }
+ if (idx == TRASH_RENAME_TRIES) {
+ ERROR("move_to_trash(%s) error: there are already %d files in the trash "
+ "with this name.\n", abs_path, TRASH_RENAME_TRIES);
+ ret = EINVAL;
+ goto done;
}
}
- if (hdfsRename(userFS, item, target)) {
- ERROR("Trying to rename %s to %s", item, target);
- return -EIO;
- }
- return 0;
-}
-
-
-int hdfsDeleteWithTrash(hdfsFS userFS, const char *path, int useTrash) {
+ if (hdfsRename(userFS, abs_path, target)) {
+ ret = errno;
+ ERROR("move_to_trash(%s): failed to rename the file to %s: error %d",
+ abs_path, target, ret);
+ goto done;
+ }
+
+ ret = 0;
+done:
+ if ((ret != 0) && (ret != ALREADY_IN_TRASH_ERR)) {
+ ERROR("move_to_trash(%s) failed with error %d", abs_path, ret);
+ }
+ free(pcomp);
+ free(parent_dir);
+ free(trash_base);
+ free(target_dir);
+ free(target);
+ return ret;
+}
- // move the file to the trash if this is enabled and its not actually in the trash.
- if (useTrash && strncmp(path, TrashPrefixDir, strlen(TrashPrefixDir)) != 0) {
- int ret= move_to_trash(path, userFS);
- return ret;
+int hdfsDeleteWithTrash(hdfsFS userFS, const char *path, int useTrash)
+{
+ int tried_to_move_to_trash = 0;
+ if (useTrash) {
+ tried_to_move_to_trash = 1;
+ if (move_to_trash(path, userFS) == 0) {
+ return 0;
+ }
}
-
if (hdfsDelete(userFS, path, 1)) {
- ERROR("Trying to delete the file %s", path);
- return -EIO;
+ int err = errno;
+ if (err < 0) {
+ err = -err;
+ }
+ ERROR("hdfsDeleteWithTrash(%s): hdfsDelete failed: error %d.",
+ path, err);
+ return -err;
+ }
+ if (tried_to_move_to_trash) {
+ ERROR("hdfsDeleteWithTrash(%s): deleted the file instead.\n", path);
}
-
return 0;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Mon May 19 19:29:30 2014
@@ -1318,6 +1318,17 @@
</property>
<property>
+ <name>dfs.nfs.allow.insecure.ports</name>
+ <value>true</value>
+ <description>
+ When set to false, client connections originating from unprivileged ports
+ (those above 1023) will be rejected. This is to ensure that clients
+ connecting to this NFS Gateway must have had root privilege on the machine
+ where they're connecting from.
+ </description>
+</property>
+
+<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
<description>
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1594887-1595999
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1594887-1595999
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1594887-1595999
Propchange: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1588992-1595999
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java Mon May 19 19:29:30 2014
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
@@ -37,11 +38,15 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -327,4 +332,70 @@ public class TestFileAppend4 {
cluster.shutdown();
}
}
+
+ /**
+ * Test that an append with no locations fails with an exception
+ * showing insufficient locations.
+ */
+ @Test(timeout = 60000)
+ public void testAppendInsufficientLocations() throws Exception {
+ Configuration conf = new Configuration();
+
+ // lower heartbeat interval for fast recognition of DN
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ 1000);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000);
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4)
+ .build();
+ DistributedFileSystem fileSystem = null;
+ try {
+ // create a file with replication 3
+ fileSystem = cluster.getFileSystem();
+ Path f = new Path("/testAppend");
+ FSDataOutputStream create = fileSystem.create(f, (short) 2);
+ create.write("/testAppend".getBytes());
+ create.close();
+
+ // Check for replications
+ DFSTestUtil.waitReplication(fileSystem, f, (short) 2);
+
+ // Shut down all DNs that have the last block location for the file
+ LocatedBlocks lbs = fileSystem.dfs.getNamenode().
+ getBlockLocations("/testAppend", 0, Long.MAX_VALUE);
+ List<DataNode> dnsOfCluster = cluster.getDataNodes();
+ DatanodeInfo[] dnsWithLocations = lbs.getLastLocatedBlock().
+ getLocations();
+ for( DataNode dn : dnsOfCluster) {
+ for(DatanodeInfo loc: dnsWithLocations) {
+ if(dn.getDatanodeId().equals(loc)){
+ dn.shutdown();
+ DFSTestUtil.waitForDatanodeDeath(dn);
+ }
+ }
+ }
+
+ // Wait till 0 replication is recognized
+ DFSTestUtil.waitReplication(fileSystem, f, (short) 0);
+
+ // Append to the file, at this state there are 3 live DNs but none of them
+ // have the block.
+ try{
+ fileSystem.append(f);
+ fail("Append should fail because insufficient locations");
+ } catch (IOException e){
+ LOG.info("Expected exception: ", e);
+ }
+ FSDirectory dir = cluster.getNamesystem().getFSDirectory();
+ final INodeFile inode = INodeFile.
+ valueOf(dir.getINode("/testAppend"), "/testAppend");
+ assertTrue("File should remain closed", !inode.isUnderConstruction());
+ } finally {
+ if (null != fileSystem) {
+ fileSystem.close();
+ }
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Mon May 19 19:29:30 2014
@@ -22,8 +22,9 @@ import static org.junit.Assert.assertEqu
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
@@ -39,6 +40,9 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup;
import org.apache.hadoop.net.NetworkTopology;
@@ -53,7 +57,7 @@ public class TestBalancerWithNodeGroup {
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestBalancerWithNodeGroup");
- final private static long CAPACITY = 6000L;
+ final private static long CAPACITY = 5000L;
final private static String RACK0 = "/rack0";
final private static String RACK1 = "/rack1";
final private static String NODEGROUP0 = "/nodegroup0";
@@ -77,6 +81,7 @@ public class TestBalancerWithNodeGroup {
static Configuration createConf() {
Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
NetworkTopologyWithNodeGroup.class.getName());
conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
@@ -191,6 +196,19 @@ public class TestBalancerWithNodeGroup {
LOG.info("Rebalancing with default factor.");
}
+ private Set<ExtendedBlock> getBlocksOnRack(List<LocatedBlock> blks, String rack) {
+ Set<ExtendedBlock> ret = new HashSet<ExtendedBlock>();
+ for (LocatedBlock blk : blks) {
+ for (DatanodeInfo di : blk.getLocations()) {
+ if (rack.equals(NetworkTopology.getFirstHalf(di.getNetworkLocation()))) {
+ ret.add(blk.getBlock());
+ break;
+ }
+ }
+ }
+ return ret;
+ }
+
/**
* Create a cluster with even distribution, and a new empty node is added to
* the cluster, then test rack locality for balancer policy.
@@ -220,9 +238,14 @@ public class TestBalancerWithNodeGroup {
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
- TestBalancer.createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
+ long length = totalUsedSpace / numOfDatanodes;
+ TestBalancer.createFile(cluster, filePath, length,
(short) numOfDatanodes, 0);
+ LocatedBlocks lbs = client.getBlockLocations(filePath.toUri().getPath(), 0,
+ length);
+ Set<ExtendedBlock> before = getBlocksOnRack(lbs.getLocatedBlocks(), RACK0);
+
long newCapacity = CAPACITY;
String newRack = RACK1;
String newNodeGroup = NODEGROUP2;
@@ -235,22 +258,9 @@ public class TestBalancerWithNodeGroup {
// run balancer and validate results
runBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
- DatanodeInfo[] datanodeReport =
- client.getDatanodeReport(DatanodeReportType.ALL);
-
- Map<String, Integer> rackToUsedCapacity = new HashMap<String, Integer>();
- for (DatanodeInfo datanode: datanodeReport) {
- String rack = NetworkTopology.getFirstHalf(datanode.getNetworkLocation());
- int usedCapacity = (int) datanode.getDfsUsed();
-
- if (rackToUsedCapacity.get(rack) != null) {
- rackToUsedCapacity.put(rack, usedCapacity + rackToUsedCapacity.get(rack));
- } else {
- rackToUsedCapacity.put(rack, usedCapacity);
- }
- }
- assertEquals(rackToUsedCapacity.size(), 2);
- assertEquals(rackToUsedCapacity.get(RACK0), rackToUsedCapacity.get(RACK1));
+ lbs = client.getBlockLocations(filePath.toUri().getPath(), 0, length);
+ Set<ExtendedBlock> after = getBlocksOnRack(lbs.getLocatedBlocks(), RACK0);
+ assertEquals(before, after);
} finally {
cluster.shutdown();
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Mon May 19 19:29:30 2014
@@ -477,6 +477,12 @@ public class TestCacheDirectives {
iter = dfs.listCacheDirectives(
new CacheDirectiveInfo.Builder().setPool("pool2").build());
validateListAll(iter, betaId);
+ iter = dfs.listCacheDirectives(
+ new CacheDirectiveInfo.Builder().setId(alphaId2).build());
+ validateListAll(iter, alphaId2);
+ iter = dfs.listCacheDirectives(
+ new CacheDirectiveInfo.Builder().setId(relativeId).build());
+ validateListAll(iter, relativeId);
dfs.removeCacheDirective(betaId);
iter = dfs.listCacheDirectives(
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java Mon May 19 19:29:30 2014
@@ -129,4 +129,44 @@ public class TestHostsFiles {
cluster.shutdown();
}
}
+
+ @Test
+ public void testHostsIncludeForDeadCount() throws Exception {
+ Configuration conf = getConf();
+
+ // Configure an excludes file
+ FileSystem localFileSys = FileSystem.getLocal(conf);
+ Path workingDir = localFileSys.getWorkingDirectory();
+ Path dir = new Path(workingDir, "build/test/data/temp/decommission");
+ Path excludeFile = new Path(dir, "exclude");
+ Path includeFile = new Path(dir, "include");
+ assertTrue(localFileSys.mkdirs(dir));
+ StringBuilder includeHosts = new StringBuilder();
+ includeHosts.append("localhost:52").append("\n").append("127.0.0.1:7777")
+ .append("\n");
+ DFSTestUtil.writeFile(localFileSys, excludeFile, "");
+ DFSTestUtil.writeFile(localFileSys, includeFile, includeHosts.toString());
+ conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath());
+ conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
+
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final FSNamesystem ns = cluster.getNameNode().getNamesystem();
+ assertTrue(ns.getNumDeadDataNodes() == 2);
+ assertTrue(ns.getNumLiveDataNodes() == 0);
+
+ // Testing using MBeans
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=FSNamesystemState");
+ String nodes = mbs.getAttribute(mxbeanName, "NumDeadDataNodes") + "";
+ assertTrue((Integer) mbs.getAttribute(mxbeanName, "NumDeadDataNodes") == 2);
+ assertTrue((Integer) mbs.getAttribute(mxbeanName, "NumLiveDataNodes") == 0);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml?rev=1596000&r1=1595999&r2=1596000&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml Mon May 19 19:29:30 2014
@@ -519,5 +519,29 @@
</comparator>
</comparators>
</test>
+
+ <test> <!--Tested -->
+ <description>Testing listing a single cache directive</description>
+ <test-commands>
+ <cache-admin-command>-addPool pool1</cache-admin-command>
+ <cache-admin-command>-addDirective -path /foo -pool pool1 -ttl 2d</cache-admin-command>
+ <cache-admin-command>-addDirective -path /bar -pool pool1 -ttl 24h</cache-admin-command>
+ <cache-admin-command>-addDirective -path /baz -replication 2 -pool pool1 -ttl 60m</cache-admin-command>
+ <cache-admin-command>-listDirectives -stats -id 30</cache-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <cache-admin-command>-removePool pool1</cache-admin-command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Found 1 entry</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>30 pool1 1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>