You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/10/11 08:14:40 UTC
svn commit: r1396918 [2/3] - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs/ hadoop-hdfs/src/ h...
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Oct 11 06:14:26 2012
@@ -39,6 +39,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT;
@@ -209,6 +211,7 @@ public class DFSClient implements java.i
final int writePacketSize;
final int socketTimeout;
final int socketCacheCapacity;
+ final long socketCacheExpiry;
/** Wait time window (in msec) if BlockMissingException is caught */
final int timeWindow;
final int nCachedConnRetry;
@@ -257,6 +260,8 @@ public class DFSClient implements java.i
taskId = conf.get("mapreduce.task.attempt.id", "NONMAPREDUCE");
socketCacheCapacity = conf.getInt(DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY,
DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT);
+ socketCacheExpiry = conf.getLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY,
+ DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT);
prefetchSize = conf.getLong(DFS_CLIENT_READ_PREFETCH_SIZE_KEY,
10 * defaultBlockSize);
timeWindow = conf
@@ -427,7 +432,7 @@ public class DFSClient implements java.i
Joiner.on(',').join(localInterfaceAddrs) + "]");
}
- this.socketCache = new SocketCache(dfsClientConf.socketCacheCapacity);
+ this.socketCache = SocketCache.getInstance(dfsClientConf.socketCacheCapacity, dfsClientConf.socketCacheExpiry);
}
/**
@@ -641,7 +646,6 @@ public class DFSClient implements java.i
void abort() {
clientRunning = false;
closeAllFilesBeingWritten(true);
- socketCache.clear();
try {
// remove reference to this client and stop the renewer,
@@ -688,7 +692,6 @@ public class DFSClient implements java.i
public synchronized void close() throws IOException {
if(clientRunning) {
closeAllFilesBeingWritten(false);
- socketCache.clear();
clientRunning = false;
getLeaseRenewer().closeClient(this);
// close connections to the namenode
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Oct 11 06:14:26 2012
@@ -74,6 +74,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY = "dfs.client.failover.connection.retries.on.timeouts";
public static final int DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0;
+ public static final String DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY = "dfs.client.socketcache.expiryMsec";
+ public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 2 * 60 * 1000;
public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address";
public static final String DFS_NAMENODE_BACKUP_ADDRESS_DEFAULT = "localhost:50100";
public static final String DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY = "dfs.namenode.backup.http-address";
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Thu Oct 11 06:14:26 2012
@@ -254,6 +254,9 @@ public class HftpFileSystem extends File
", assuming security is disabled");
return null;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exception getting delegation token", e);
+ }
throw e;
}
for (Token<? extends TokenIdentifier> t : c.getAllTokens()) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SocketCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SocketCache.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SocketCache.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SocketCache.java Thu Oct 11 06:14:26 2012
@@ -26,51 +26,131 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
+import java.io.IOException;
import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedListMultimap;
import org.apache.commons.logging.Log;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.StringUtils;
/**
- * A cache of sockets.
+ * A cache of input stream sockets to Data Node.
*/
class SocketCache {
- static final Log LOG = LogFactory.getLog(SocketCache.class);
+ private static final Log LOG = LogFactory.getLog(SocketCache.class);
- private final LinkedListMultimap<SocketAddress, SocketAndStreams> multimap;
- private final int capacity;
+ @InterfaceAudience.Private
+ static class SocketAndStreams implements Closeable {
+ public final Socket sock;
+ public final IOStreamPair ioStreams;
+ long createTime;
+
+ public SocketAndStreams(Socket s, IOStreamPair ioStreams) {
+ this.sock = s;
+ this.ioStreams = ioStreams;
+ this.createTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public void close() {
+ if (ioStreams != null) {
+ IOUtils.closeStream(ioStreams.in);
+ IOUtils.closeStream(ioStreams.out);
+ }
+ IOUtils.closeSocket(sock);
+ }
- /**
- * Create a SocketCache with the given capacity.
- * @param capacity Max cache size.
- */
- public SocketCache(int capacity) {
- multimap = LinkedListMultimap.create();
- this.capacity = capacity;
- if (capacity <= 0) {
- LOG.debug("SocketCache disabled in configuration.");
+ public long getCreateTime() {
+ return this.createTime;
}
}
+ private Daemon daemon;
+ /** A map for per user per datanode. */
+ private static LinkedListMultimap<SocketAddress, SocketAndStreams> multimap =
+ LinkedListMultimap.create();
+ private static int capacity;
+ private static long expiryPeriod;
+ private static SocketCache scInstance = new SocketCache();
+ private static boolean isInitedOnce = false;
+
+ public static synchronized SocketCache getInstance(int c, long e) {
+ // capacity is only initialized once
+ if (isInitedOnce == false) {
+ capacity = c;
+ expiryPeriod = e;
+
+ if (capacity == 0 ) {
+ LOG.info("SocketCache disabled.");
+ }
+ else if (expiryPeriod == 0) {
+ throw new IllegalStateException("Cannot initialize expiryPeriod to " +
+ expiryPeriod + "when cache is enabled.");
+ }
+ isInitedOnce = true;
+ } else { //already initialized once
+ if (capacity != c || expiryPeriod != e) {
+ LOG.info("capacity and expiry periods already set to " + capacity +
+ " and " + expiryPeriod + " respectively. Cannot set it to " + c +
+ " and " + e);
+ }
+ }
+
+ return scInstance;
+ }
+
+ private boolean isDaemonStarted() {
+ return (daemon == null)? false: true;
+ }
+
+ private synchronized void startExpiryDaemon() {
+ // start daemon only if not already started
+ if (isDaemonStarted() == true) {
+ return;
+ }
+
+ daemon = new Daemon(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ SocketCache.this.run();
+ } catch(InterruptedException e) {
+ //noop
+ } finally {
+ SocketCache.this.clear();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(SocketCache.this);
+ }
+ });
+ daemon.start();
+ }
+
/**
* Get a cached socket to the given address.
* @param remote Remote address the socket is connected to.
* @return A socket with unknown state, possibly closed underneath. Or null.
*/
public synchronized SocketAndStreams get(SocketAddress remote) {
+
if (capacity <= 0) { // disabled
return null;
}
-
- List<SocketAndStreams> socklist = multimap.get(remote);
- if (socklist == null) {
+
+ List<SocketAndStreams> sockStreamList = multimap.get(remote);
+ if (sockStreamList == null) {
return null;
}
- Iterator<SocketAndStreams> iter = socklist.iterator();
+ Iterator<SocketAndStreams> iter = sockStreamList.iterator();
while (iter.hasNext()) {
SocketAndStreams candidate = iter.next();
iter.remove();
@@ -86,14 +166,16 @@ class SocketCache {
* @param sock socket not used by anyone.
*/
public synchronized void put(Socket sock, IOStreamPair ioStreams) {
+
+ Preconditions.checkNotNull(sock);
SocketAndStreams s = new SocketAndStreams(sock, ioStreams);
if (capacity <= 0) {
// Cache disabled.
s.close();
return;
}
-
- Preconditions.checkNotNull(sock);
+
+ startExpiryDaemon();
SocketAddress remoteAddr = sock.getRemoteSocketAddress();
if (remoteAddr == null) {
@@ -106,7 +188,7 @@ class SocketCache {
if (capacity == multimap.size()) {
evictOldest();
}
- multimap.put(remoteAddr, new SocketAndStreams(sock, ioStreams));
+ multimap.put(remoteAddr, s);
}
public synchronized int size() {
@@ -114,13 +196,34 @@ class SocketCache {
}
/**
+ * Evict and close sockets older than expiry period from the cache.
+ */
+ private synchronized void evictExpired(long expiryPeriod) {
+ while (multimap.size() != 0) {
+ Iterator<Entry<SocketAddress, SocketAndStreams>> iter =
+ multimap.entries().iterator();
+ Entry<SocketAddress, SocketAndStreams> entry = iter.next();
+ // if oldest socket expired, remove it
+ if (entry == null ||
+ System.currentTimeMillis() - entry.getValue().getCreateTime() <
+ expiryPeriod) {
+ break;
+ }
+ iter.remove();
+ SocketAndStreams s = entry.getValue();
+ s.close();
+ }
+ }
+
+ /**
* Evict the oldest entry in the cache.
*/
private synchronized void evictOldest() {
Iterator<Entry<SocketAddress, SocketAndStreams>> iter =
multimap.entries().iterator();
if (!iter.hasNext()) {
- throw new IllegalStateException("Cannot evict from empty cache!");
+ throw new IllegalStateException("Cannot evict from empty cache! " +
+ "capacity: " + capacity);
}
Entry<SocketAddress, SocketAndStreams> entry = iter.next();
iter.remove();
@@ -129,38 +232,31 @@ class SocketCache {
}
/**
- * Empty the cache, and close all sockets.
+ * Periodically check in the cache and expire the entries
+ * older than expiryPeriod minutes
*/
- public synchronized void clear() {
- for (SocketAndStreams s : multimap.values()) {
- s.close();
+ private void run() throws InterruptedException {
+ for(long lastExpiryTime = System.currentTimeMillis();
+ !Thread.interrupted();
+ Thread.sleep(expiryPeriod)) {
+ final long elapsed = System.currentTimeMillis() - lastExpiryTime;
+ if (elapsed >= expiryPeriod) {
+ evictExpired(expiryPeriod);
+ lastExpiryTime = System.currentTimeMillis();
+ }
}
- multimap.clear();
- }
-
- @Override
- protected void finalize() {
clear();
+ throw new InterruptedException("Daemon Interrupted");
}
-
- @InterfaceAudience.Private
- static class SocketAndStreams implements Closeable {
- public final Socket sock;
- public final IOStreamPair ioStreams;
-
- public SocketAndStreams(Socket s, IOStreamPair ioStreams) {
- this.sock = s;
- this.ioStreams = ioStreams;
- }
-
- @Override
- public void close() {
- if (ioStreams != null) {
- IOUtils.closeStream(ioStreams.in);
- IOUtils.closeStream(ioStreams.out);
- }
- IOUtils.closeSocket(sock);
+
+ /**
+ * Empty the cache, and close all sockets.
+ */
+ private synchronized void clear() {
+ for (SocketAndStreams sockAndStream : multimap.values()) {
+ sockAndStream.close();
}
+ multimap.clear();
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Thu Oct 11 06:14:26 2012
@@ -612,7 +612,8 @@ public class DatanodeManager {
+ " storage " + nodeReg.getStorageID());
DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getStorageID());
- DatanodeDescriptor nodeN = getDatanodeByHost(nodeReg.getXferAddr());
+ DatanodeDescriptor nodeN = host2DatanodeMap.getDatanodeByXferAddr(
+ nodeReg.getIpAddr(), nodeReg.getXferPort());
if (nodeN != null && nodeN != nodeS) {
NameNode.LOG.info("BLOCK* NameSystem.registerDatanode: "
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java Thu Oct 11 06:14:26 2012
@@ -159,6 +159,35 @@ class Host2NodesMap {
}
}
+ /**
+ * Find data node by its transfer address
+ *
+ * @return DatanodeDescriptor if found or null otherwise
+ */
+ public DatanodeDescriptor getDatanodeByXferAddr(String ipAddr,
+ int xferPort) {
+ if (ipAddr==null) {
+ return null;
+ }
+
+ hostmapLock.readLock().lock();
+ try {
+ DatanodeDescriptor[] nodes = map.get(ipAddr);
+ // no entry
+ if (nodes== null) {
+ return null;
+ }
+ for(DatanodeDescriptor containedNode:nodes) {
+ if (xferPort == containedNode.getXferPort()) {
+ return containedNode;
+ }
+ }
+ return null;
+ } finally {
+ hostmapLock.readLock().unlock();
+ }
+ }
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder(getClass().getSimpleName())
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Thu Oct 11 06:14:26 2012
@@ -276,6 +276,9 @@ public class JspHelper {
FIELD_PERCENT_REMAINING = 9,
FIELD_ADMIN_STATE = 10,
FIELD_DECOMMISSIONED = 11,
+ FIELD_BLOCKPOOL_USED = 12,
+ FIELD_PERBLOCKPOOL_USED = 13,
+ FIELD_FAILED_VOLUMES = 14,
SORT_ORDER_ASC = 1,
SORT_ORDER_DSC = 2;
@@ -303,6 +306,12 @@ public class JspHelper {
sortField = FIELD_ADMIN_STATE;
} else if (field.equals("decommissioned")) {
sortField = FIELD_DECOMMISSIONED;
+ } else if (field.equals("bpused")) {
+ sortField = FIELD_BLOCKPOOL_USED;
+ } else if (field.equals("pcbpused")) {
+ sortField = FIELD_PERBLOCKPOOL_USED;
+ } else if (field.equals("volfails")) {
+ sortField = FIELD_FAILED_VOLUMES;
} else {
sortField = FIELD_NAME;
}
@@ -361,6 +370,18 @@ public class JspHelper {
case FIELD_NAME:
ret = d1.getHostName().compareTo(d2.getHostName());
break;
+ case FIELD_BLOCKPOOL_USED:
+ dlong = d1.getBlockPoolUsed() - d2.getBlockPoolUsed();
+ ret = (dlong < 0) ? -1 : ((dlong > 0) ? 1 : 0);
+ break;
+ case FIELD_PERBLOCKPOOL_USED:
+ ddbl = d1.getBlockPoolUsedPercent() - d2.getBlockPoolUsedPercent();
+ ret = (ddbl < 0) ? -1 : ((ddbl > 0) ? 1 : 0);
+ break;
+ case FIELD_FAILED_VOLUMES:
+ int dint = d1.getVolumeFailures() - d2.getVolumeFailures();
+ ret = (dint < 0) ? -1 : ((dint > 0) ? 1 : 0);
+ break;
}
return (sortOrder == SORT_ORDER_DSC) ? -ret : ret;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Oct 11 06:14:26 2012
@@ -4059,7 +4059,10 @@ public class FSNamesystem implements Nam
return "Safe mode is OFF.";
String leaveMsg = "";
if (areResourcesLow()) {
- leaveMsg = "Resources are low on NN. Safe mode must be turned off manually";
+ leaveMsg = "Resources are low on NN. "
+ + "Please add or free up more resources then turn off safe mode manually. "
+ + "NOTE: If you turn off safe mode before adding resources, "
+ + "the NN will immediately return to safe mode.";
} else {
leaveMsg = "Safe mode will be turned off automatically";
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Oct 11 06:14:26 2012
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -510,7 +511,7 @@ public class NameNode {
stopHttpServer();
}
- private void startTrashEmptier(Configuration conf) throws IOException {
+ private void startTrashEmptier(final Configuration conf) throws IOException {
long trashInterval =
conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT);
if (trashInterval == 0) {
@@ -519,7 +520,18 @@ public class NameNode {
throw new IOException("Cannot start tresh emptier with negative interval."
+ " Set " + FS_TRASH_INTERVAL_KEY + " to a positive value.");
}
- this.emptier = new Thread(new Trash(conf).getEmptier(), "Trash Emptier");
+
+ // This may be called from the transitionToActive code path, in which
+ // case the current user is the administrator, not the NN. The trash
+ // emptier needs to run as the NN. See HDFS-3972.
+ FileSystem fs = SecurityUtil.doAsLoginUser(
+ new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return FileSystem.get(conf);
+ }
+ });
+ this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier");
this.emptier.setDaemon(true);
this.emptier.start();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Thu Oct 11 06:14:26 2012
@@ -107,6 +107,10 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
SecurityUtil.getServerPrincipal(principalInConf,
bindAddress.getHostName()));
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.error("WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
+ "' is not set.");
}
String httpKeytab = conf.get(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
@@ -117,6 +121,10 @@ public class NameNodeHttpServer {
params.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
httpKeytab);
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.error("WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY +
+ "' is not set.");
}
return params;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Thu Oct 11 06:14:26 2012
@@ -78,6 +78,7 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
/**********************************************************
@@ -122,6 +123,8 @@ public class SecondaryNameNode implement
private CheckpointConf checkpointConf;
private FSNamesystem namesystem;
+ private Thread checkpointThread;
+
@Override
public String toString() {
@@ -277,6 +280,15 @@ public class SecondaryNameNode implement
*/
public void shutdown() {
shouldRun = false;
+ if (checkpointThread != null) {
+ checkpointThread.interrupt();
+ try {
+ checkpointThread.join(10000);
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted waiting to join on checkpointer thread");
+ Thread.currentThread().interrupt(); // maintain status
+ }
+ }
try {
if (infoServer != null) infoServer.stop();
} catch (Exception e) {
@@ -586,12 +598,20 @@ public class SecondaryNameNode implement
terminate(ret);
}
- // Create a never ending deamon
- Daemon checkpointThread = new Daemon(secondary);
- checkpointThread.start();
+ secondary.startCheckpointThread();
}
+ public void startCheckpointThread() {
+ Preconditions.checkState(checkpointThread == null,
+ "Should not already have a thread");
+ Preconditions.checkState(shouldRun, "shouldRun should be true");
+
+ checkpointThread = new Daemon(this);
+ checkpointThread.start();
+ }
+
+
/**
* Container for parsed command-line options.
*/
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Thu Oct 11 06:14:26 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.tools;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -53,6 +54,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.util.StringUtils;
@@ -80,7 +82,7 @@ public class DFSAdmin extends FsShell {
super(fs.getConf());
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not a distributed file system");
+ " is not an HDFS file system");
}
this.dfs = (DistributedFileSystem)fs;
}
@@ -284,7 +286,7 @@ public class DFSAdmin extends FsShell {
FileSystem fs = getFS();
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not a distributed file system");
+ " is not an HDFS file system");
}
return (DistributedFileSystem)fs;
}
@@ -511,11 +513,17 @@ public class DFSAdmin extends FsShell {
* @return an exit code indicating success or failure.
* @throws IOException
*/
- public int fetchImage(String[] argv, int idx) throws IOException {
- String infoServer = DFSUtil.getInfoServer(
+ public int fetchImage(final String[] argv, final int idx) throws IOException {
+ final String infoServer = DFSUtil.getInfoServer(
HAUtil.getAddressOfActive(getDFS()), getConf(), false);
- TransferFsImage.downloadMostRecentImageToDirectory(infoServer,
- new File(argv[idx]));
+ SecurityUtil.doAsCurrentUser(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ TransferFsImage.downloadMostRecentImageToDirectory(infoServer,
+ new File(argv[idx]));
+ return null;
+ }
+ });
return 0;
}
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c Thu Oct 11 06:14:26 2012
@@ -67,6 +67,25 @@ static const struct ExceptionInfo gExcep
};
+void getExceptionInfo(const char *excName, int noPrintFlags,
+ int *excErrno, int *shouldPrint)
+{
+ int i;
+
+ for (i = 0; i < EXCEPTION_INFO_LEN; i++) {
+ if (strstr(gExceptionInfo[i].name, excName)) {
+ break;
+ }
+ }
+ if (i < EXCEPTION_INFO_LEN) {
+ *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags);
+ *excErrno = gExceptionInfo[i].excErrno;
+ } else {
+ *shouldPrint = 1;
+ *excErrno = EINTERNAL;
+ }
+}
+
int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags,
const char *fmt, va_list ap)
{
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h Thu Oct 11 06:14:26 2012
@@ -65,6 +65,21 @@
#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10
/**
+ * Get information about an exception.
+ *
+ * @param excName The Exception name.
+ * This is a Java class name in JNI format.
+ * @param noPrintFlags Flags which determine which exceptions we should NOT
+ * print.
+ * @param excErrno (out param) The POSIX error number associated with the
+ * exception.
+ * @param shouldPrint (out param) Nonzero if we should print this exception,
+ * based on the noPrintFlags and its name.
+ */
+void getExceptionInfo(const char *excName, int noPrintFlags,
+ int *excErrno, int *shouldPrint);
+
+/**
* Print out information about an exception and free it.
*
* @param env The JNI environment
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Thu Oct 11 06:14:26 2012
@@ -41,11 +41,34 @@
</property>
<property>
+ <name>dfs.namenode.rpc-address</name>
+ <value></value>
+ <description>
+ RPC address that handles all clients requests. In the case of HA/Federation where multiple namenodes exist,
+ the name service id is added to the name e.g. dfs.namenode.rpc-address.ns1
+ dfs.namenode.rpc-address.EXAMPLENAMESERVICE
+ The value of this property will take the form of hdfs://nn-host1:rpc-port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value></value>
+ <description>
+ RPC address for HDFS Services communication. BackupNode, Datanodes and all other services should be
+ connecting to this address if it is configured. In the case of HA/Federation where multiple namenodes exist,
+ the name service id is added to the name e.g. dfs.namenode.servicerpc-address.ns1
+ dfs.namenode.rpc-address.EXAMPLENAMESERVICE
+ The value of this property will take the form of hdfs://nn-host1:rpc-port.
+ If the value of this property is unset the value of dfs.namenode.rpc-address will be used as the default.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.secondary.http-address</name>
<value>0.0.0.0:50090</value>
<description>
The secondary namenode http server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -54,7 +77,6 @@
<value>0.0.0.0:50010</value>
<description>
The datanode server address and port for data transfer.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -63,7 +85,6 @@
<value>0.0.0.0:50075</value>
<description>
The datanode http server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -72,7 +93,6 @@
<value>0.0.0.0:50020</value>
<description>
The datanode ipc server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -87,7 +107,6 @@
<value>0.0.0.0:50070</value>
<description>
The address and the base port where the dfs namenode web ui will listen on.
- If the port is 0 then the server will start on a free port.
</description>
</property>
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1390199-1396916
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1390199-1396916
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1390199-1396916
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Thu Oct 11 06:14:26 2012
@@ -211,27 +211,40 @@ public class DFSTestUtil {
public static void createFile(FileSystem fs, Path fileName, long fileLen,
short replFactor, long seed) throws IOException {
+ createFile(fs, fileName, 1024, fileLen, fs.getDefaultBlockSize(fileName),
+ replFactor, seed);
+ }
+
+ public static void createFile(FileSystem fs, Path fileName, int bufferLen,
+ long fileLen, long blockSize, short replFactor, long seed)
+ throws IOException {
+ assert bufferLen > 0;
if (!fs.mkdirs(fileName.getParent())) {
throw new IOException("Mkdirs failed to create " +
fileName.getParent().toString());
}
FSDataOutputStream out = null;
try {
- out = fs.create(fileName, replFactor);
- byte[] toWrite = new byte[1024];
- Random rb = new Random(seed);
- long bytesToWrite = fileLen;
- while (bytesToWrite>0) {
- rb.nextBytes(toWrite);
- int bytesToWriteNext = (1024<bytesToWrite)?1024:(int)bytesToWrite;
-
- out.write(toWrite, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
+ out = fs.create(fileName, true, fs.getConf()
+ .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ replFactor, blockSize);
+ if (fileLen > 0) {
+ byte[] toWrite = new byte[bufferLen];
+ Random rb = new Random(seed);
+ long bytesToWrite = fileLen;
+ while (bytesToWrite>0) {
+ rb.nextBytes(toWrite);
+ int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen
+ : (int) bytesToWrite;
+
+ out.write(toWrite, 0, bytesToWriteNext);
+ bytesToWrite -= bytesToWriteNext;
+ }
}
- out.close();
- out = null;
} finally {
- IOUtils.closeStream(out);
+ if (out != null) {
+ out.close();
+ }
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Thu Oct 11 06:14:26 2012
@@ -624,14 +624,20 @@ public class MiniDFSCluster {
}
federation = nnTopology.isFederated();
- createNameNodesAndSetConf(
- nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
- enableManagedDfsDirsRedundancy,
- format, operation, clusterId, conf);
-
+ try {
+ createNameNodesAndSetConf(
+ nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
+ enableManagedDfsDirsRedundancy,
+ format, operation, clusterId, conf);
+ } catch (IOException ioe) {
+ LOG.error("IOE creating namenodes. Permissions dump:\n" +
+ createPermissionsDiagnosisString(data_dir));
+ throw ioe;
+ }
if (format) {
if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) {
- throw new IOException("Cannot remove data directory: " + data_dir);
+ throw new IOException("Cannot remove data directory: " + data_dir +
+ createPermissionsDiagnosisString(data_dir));
}
}
@@ -647,6 +653,27 @@ public class MiniDFSCluster {
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
}
+ /**
+ * @return a debug string which can help diagnose an error of why
+ * a given directory might have a permissions error in the context
+ * of a test case
+ */
+ private String createPermissionsDiagnosisString(File path) {
+ StringBuilder sb = new StringBuilder();
+ while (path != null) {
+ sb.append("path '" + path + "': ").append("\n");
+ sb.append("\tabsolute:").append(path.getAbsolutePath()).append("\n");
+ sb.append("\tpermissions: ");
+ sb.append(path.isDirectory() ? "d": "-");
+ sb.append(path.canRead() ? "r" : "-");
+ sb.append(path.canWrite() ? "w" : "-");
+ sb.append(path.canExecute() ? "x" : "-");
+ sb.append("\n");
+ path = path.getParentFile();
+ }
+ return sb.toString();
+ }
+
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs,
boolean enableManagedDfsDirsRedundancy, boolean format,
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java Thu Oct 11 06:14:26 2012
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,10 +55,12 @@ public class TestConnCache {
static final int BLOCK_SIZE = 4096;
static final int FILE_SIZE = 3 * BLOCK_SIZE;
-
+ final static int CACHE_SIZE = 4;
+ final static long CACHE_EXPIRY_MS = 200;
static Configuration conf = null;
static MiniDFSCluster cluster = null;
static FileSystem fs = null;
+ static SocketCache cache;
static final Path testFile = new Path("/testConnCache.dat");
static byte authenticData[] = null;
@@ -93,6 +96,9 @@ public class TestConnCache {
public static void setupCluster() throws Exception {
final int REPLICATION_FACTOR = 1;
+ /* create a socket cache. There is only one socket cache per jvm */
+ cache = SocketCache.getInstance(CACHE_SIZE, CACHE_EXPIRY_MS);
+
util = new BlockReaderTestUtil(REPLICATION_FACTOR);
cluster = util.getCluster();
conf = util.getConf();
@@ -142,10 +148,7 @@ public class TestConnCache {
* Test the SocketCache itself.
*/
@Test
- public void testSocketCache() throws IOException {
- final int CACHE_SIZE = 4;
- SocketCache cache = new SocketCache(CACHE_SIZE);
-
+ public void testSocketCache() throws Exception {
// Make a client
InetSocketAddress nnAddr =
new InetSocketAddress("localhost", cluster.getNameNodePort());
@@ -159,6 +162,7 @@ public class TestConnCache {
DataNode dn = util.getDataNode(block);
InetSocketAddress dnAddr = dn.getXferAddress();
+
// Make some sockets to the DN
Socket[] dnSockets = new Socket[CACHE_SIZE];
for (int i = 0; i < dnSockets.length; ++i) {
@@ -166,6 +170,7 @@ public class TestConnCache {
dnAddr.getAddress(), dnAddr.getPort());
}
+
// Insert a socket to the NN
Socket nnSock = new Socket(nnAddr.getAddress(), nnAddr.getPort());
cache.put(nnSock, null);
@@ -179,7 +184,7 @@ public class TestConnCache {
assertEquals("NN socket evicted", null, cache.get(nnAddr));
assertTrue("Evicted socket closed", nnSock.isClosed());
-
+
// Lookup the DN socks
for (Socket dnSock : dnSockets) {
assertEquals("Retrieve cached sockets", dnSock, cache.get(dnAddr).sock);
@@ -189,6 +194,51 @@ public class TestConnCache {
assertEquals("Cache is empty", 0, cache.size());
}
+
+ /**
+ * Test the SocketCache expiry.
+ * Verify that socket cache entries expire after the set
+ * expiry time.
+ */
+ @Test
+ public void testSocketCacheExpiry() throws Exception {
+ // Make a client
+ InetSocketAddress nnAddr =
+ new InetSocketAddress("localhost", cluster.getNameNodePort());
+ DFSClient client = new DFSClient(nnAddr, conf);
+
+ // Find out the DN addr
+ LocatedBlock block =
+ client.getNamenode().getBlockLocations(
+ testFile.toString(), 0, FILE_SIZE)
+ .getLocatedBlocks().get(0);
+ DataNode dn = util.getDataNode(block);
+ InetSocketAddress dnAddr = dn.getXferAddress();
+
+
+ // Make some sockets to the DN and put in cache
+ Socket[] dnSockets = new Socket[CACHE_SIZE];
+ for (int i = 0; i < dnSockets.length; ++i) {
+ dnSockets[i] = client.socketFactory.createSocket(
+ dnAddr.getAddress(), dnAddr.getPort());
+ cache.put(dnSockets[i], null);
+ }
+
+ // Client side still has the sockets cached
+ assertEquals(CACHE_SIZE, client.socketCache.size());
+
+ //sleep for a second and see if it expired
+ Thread.sleep(CACHE_EXPIRY_MS + 1000);
+
+ // Client side has no sockets cached
+ assertEquals(0, client.socketCache.size());
+
+ //sleep for another second and see if
+ //the daemon thread runs fine on empty cache
+ Thread.sleep(CACHE_EXPIRY_MS + 1000);
+ }
+
+
/**
* Read a file served entirely from one DN. Seek around and read from
* different offsets. And verify that they all use the same socket.
@@ -229,33 +279,6 @@ public class TestConnCache {
in.close();
}
-
- /**
- * Test that the socket cache can be disabled by setting the capacity to
- * 0. Regression test for HDFS-3365.
- */
- @Test
- public void testDisableCache() throws IOException {
- LOG.info("Starting testDisableCache()");
-
- // Reading with the normally configured filesystem should
- // cache a socket.
- DFSTestUtil.readFile(fs, testFile);
- assertEquals(1, ((DistributedFileSystem)fs).dfs.socketCache.size());
-
- // Configure a new instance with no caching, ensure that it doesn't
- // cache anything
- Configuration confWithoutCache = new Configuration(fs.getConf());
- confWithoutCache.setInt(
- DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY, 0);
- FileSystem fsWithoutCache = FileSystem.newInstance(confWithoutCache);
- try {
- DFSTestUtil.readFile(fsWithoutCache, testFile);
- assertEquals(0, ((DistributedFileSystem)fsWithoutCache).dfs.socketCache.size());
- } finally {
- fsWithoutCache.close();
- }
- }
@AfterClass
public static void teardownCluster() throws Exception {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java Thu Oct 11 06:14:26 2012
@@ -37,7 +37,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -141,13 +140,6 @@ public class TestDataTransferProtocol {
}
}
- void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
- byte [] arr = new byte[fileLen];
- FSDataOutputStream out = fs.create(path);
- out.write(arr);
- out.close();
- }
-
void readFile(FileSystem fs, Path path, int fileLen) throws IOException {
byte [] arr = new byte[fileLen];
FSDataInputStream in = fs.open(path);
@@ -357,7 +349,9 @@ public class TestDataTransferProtocol {
int fileLen = Math.min(conf.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096), 4096);
- createFile(fileSys, file, fileLen);
+ DFSTestUtil.createFile(fileSys, file, fileLen, fileLen,
+ fileSys.getDefaultBlockSize(file),
+ fileSys.getDefaultReplication(file), 0L);
// get the first blockid for the file
final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fileSys, file);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java Thu Oct 11 06:14:26 2012
@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -92,6 +93,58 @@ public class TestDatanodeRegistration {
}
@Test
+ public void testChangeStorageID() throws Exception {
+ final String DN_IP_ADDR = "127.0.0.1";
+ final String DN_HOSTNAME = "localhost";
+ final int DN_XFER_PORT = 12345;
+ final int DN_INFO_PORT = 12346;
+ final int DN_IPC_PORT = 12347;
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build();
+ InetSocketAddress addr = new InetSocketAddress(
+ "localhost",
+ cluster.getNameNodePort());
+ DFSClient client = new DFSClient(addr, conf);
+ NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
+
+ // register a datanode
+ DatanodeID dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME,
+ "fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT);
+ long nnCTime = cluster.getNamesystem().getFSImage().getStorage()
+ .getCTime();
+ StorageInfo mockStorageInfo = mock(StorageInfo.class);
+ doReturn(nnCTime).when(mockStorageInfo).getCTime();
+ doReturn(HdfsConstants.LAYOUT_VERSION).when(mockStorageInfo)
+ .getLayoutVersion();
+ DatanodeRegistration dnReg = new DatanodeRegistration(dnId,
+ mockStorageInfo, null, VersionInfo.getVersion());
+ rpcServer.registerDatanode(dnReg);
+
+ DatanodeInfo[] report = client.datanodeReport(DatanodeReportType.ALL);
+ assertEquals("Expected a registered datanode", 1, report.length);
+
+ // register the same datanode again with a different storage ID
+ dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME,
+ "changed-fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT);
+ dnReg = new DatanodeRegistration(dnId,
+ mockStorageInfo, null, VersionInfo.getVersion());
+ rpcServer.registerDatanode(dnReg);
+
+ report = client.datanodeReport(DatanodeReportType.ALL);
+ assertEquals("Datanode with changed storage ID not recognized",
+ 1, report.length);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
public void testRegistrationWithDifferentSoftwareVersions() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "3.0.0");
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Thu Oct 11 06:14:26 2012
@@ -120,12 +120,9 @@ public class TestDistributedFileSystem {
DFSTestUtil.readFile(fileSys, p);
DFSClient client = ((DistributedFileSystem)fileSys).dfs;
- SocketCache cache = client.socketCache;
- assertEquals(1, cache.size());
fileSys.close();
- assertEquals(0, cache.size());
} finally {
if (cluster != null) {cluster.shutdown();}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java Thu Oct 11 06:14:26 2012
@@ -79,7 +79,8 @@ public class TestFileStatus {
hftpfs = cluster.getHftpFileSystem(0);
dfsClient = new DFSClient(NameNode.getAddress(conf), conf);
file1 = new Path("filestatus.dat");
- writeFile(fs, file1, 1, fileSize, blockSize);
+ DFSTestUtil.createFile(fs, file1, fileSize, fileSize, blockSize, (short) 1,
+ seed);
}
@AfterClass
@@ -87,18 +88,6 @@ public class TestFileStatus {
fs.close();
cluster.shutdown();
}
-
- private static void writeFile(FileSystem fileSys, Path name, int repl,
- int fileSize, int blockSize) throws IOException {
- // Create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true,
- HdfsConstants.IO_FILE_BUFFER_SIZE, (short)repl, (long)blockSize);
- byte[] buffer = new byte[fileSize];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
private void checkFile(FileSystem fileSys, Path name, int repl)
throws IOException, InterruptedException, TimeoutException {
@@ -218,7 +207,8 @@ public class TestFileStatus {
// create another file that is smaller than a block.
Path file2 = new Path(dir, "filestatus2.dat");
- writeFile(fs, file2, 1, blockSize/4, blockSize);
+ DFSTestUtil.createFile(fs, file2, blockSize/4, blockSize/4, blockSize,
+ (short) 1, seed);
checkFile(fs, file2, 1);
// verify file attributes
@@ -230,7 +220,8 @@ public class TestFileStatus {
// Create another file in the same directory
Path file3 = new Path(dir, "filestatus3.dat");
- writeFile(fs, file3, 1, blockSize/4, blockSize);
+ DFSTestUtil.createFile(fs, file3, blockSize/4, blockSize/4, blockSize,
+ (short) 1, seed);
checkFile(fs, file3, 1);
file3 = fs.makeQualified(file3);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java Thu Oct 11 06:14:26 2012
@@ -110,9 +110,7 @@ public class TestGetBlocks {
// do the writing but do not close the FSDataOutputStream
// in order to mimic the ongoing writing
final Path fileName = new Path("/file1");
- stm = fileSys.create(
- fileName,
- true,
+ stm = fileSys.create(fileName, true,
fileSys.getConf().getInt(
CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
(short) 3, blockSize);
@@ -180,29 +178,15 @@ public class TestGetBlocks {
final short REPLICATION_FACTOR = (short) 2;
final int DEFAULT_BLOCK_SIZE = 1024;
- final Random r = new Random();
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
REPLICATION_FACTOR).build();
try {
cluster.waitActive();
-
- // create a file with two blocks
- FileSystem fs = cluster.getFileSystem();
- FSDataOutputStream out = fs.create(new Path("/tmp.txt"),
- REPLICATION_FACTOR);
- byte[] data = new byte[1024];
long fileLen = 2 * DEFAULT_BLOCK_SIZE;
- long bytesToWrite = fileLen;
- while (bytesToWrite > 0) {
- r.nextBytes(data);
- int bytesToWriteNext = (1024 < bytesToWrite) ? 1024
- : (int) bytesToWrite;
- out.write(data, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
- }
- out.close();
+ DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/tmp.txt"),
+ fileLen, REPLICATION_FACTOR, 0L);
// get blocks & data nodes
List<LocatedBlock> locatedBlocks;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Thu Oct 11 06:14:26 2012
@@ -41,6 +41,8 @@ public class TestHftpDelegationToken {
@Test
public void testHdfsDelegationToken() throws Exception {
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
final Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
@@ -265,4 +267,4 @@ public class TestHftpDelegationToken {
@Override
protected void initDelegationToken() throws IOException {}
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java Thu Oct 11 06:14:26 2012
@@ -102,9 +102,15 @@ public class TestHftpFileSystem {
@AfterClass
public static void tearDown() throws IOException {
- hdfs.close();
- hftpFs.close();
- cluster.shutdown();
+ if (hdfs != null) {
+ hdfs.close();
+ }
+ if (hftpFs != null) {
+ hftpFs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
/**
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Thu Oct 11 06:14:26 2012
@@ -53,19 +53,23 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf);
- HttpURLConnection conn = fs.openConnection("/", "");
- timedout = false;
try {
- // this will consume the only slot in the backlog
- conn.getInputStream();
- } catch (SocketTimeoutException ste) {
- timedout = true;
- assertEquals("Read timed out", ste.getMessage());
+ HttpURLConnection conn = fs.openConnection("/", "");
+ timedout = false;
+ try {
+ // this will consume the only slot in the backlog
+ conn.getInputStream();
+ } catch (SocketTimeoutException ste) {
+ timedout = true;
+ assertEquals("Read timed out", ste.getMessage());
+ } finally {
+ if (conn != null) conn.disconnect();
+ }
+ assertTrue("read timedout", timedout);
+ assertTrue("connect timedout", checkConnectTimeout(fs, false));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
- assertTrue("read timedout", timedout);
- assertTrue("connect timedout", checkConnectTimeout(fs, false));
}
@Test
@@ -79,20 +83,24 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf);
- HttpURLConnection conn = null;
- timedout = false;
try {
- // this will consume the only slot in the backlog
- conn = fs.openConnection("/", "");
- } catch (SocketTimeoutException ste) {
- // SSL expects a negotiation, so it will timeout on read, unlike hftp
- timedout = true;
- assertEquals("Read timed out", ste.getMessage());
+ HttpURLConnection conn = null;
+ timedout = false;
+ try {
+ // this will consume the only slot in the backlog
+ conn = fs.openConnection("/", "");
+ } catch (SocketTimeoutException ste) {
+ // SSL expects a negotiation, so it will timeout on read, unlike hftp
+ timedout = true;
+ assertEquals("Read timed out", ste.getMessage());
+ } finally {
+ if (conn != null) conn.disconnect();
+ }
+ assertTrue("ssl read connect timedout", timedout);
+ assertTrue("connect timedout", checkConnectTimeout(fs, true));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
- assertTrue("ssl read connect timedout", timedout);
- assertTrue("connect timedout", checkConnectTimeout(fs, true));
}
private boolean checkConnectTimeout(HftpFileSystem fs, boolean ignoreReadTimeout)
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java Thu Oct 11 06:14:26 2012
@@ -52,22 +52,6 @@ public class TestInjectionForSimulatedSt
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestInjectionForSimulatedStorage");
-
- private void writeFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- // create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- (short) repl, blockSize);
- byte[] buffer = new byte[filesize];
- for (int i=0; i<buffer.length; i++) {
- buffer[i] = '1';
- }
- stm.write(buffer);
- stm.close();
- }
-
- // Waits for all of the blocks to have expected replication
// Waits for all of the blocks to have expected replication
private void waitForBlockReplication(String filename,
@@ -149,7 +133,8 @@ public class TestInjectionForSimulatedSt
cluster.getNameNodePort()),
conf);
- writeFile(cluster.getFileSystem(), testPath, numDataNodes);
+ DFSTestUtil.createFile(cluster.getFileSystem(), testPath, filesize,
+ filesize, blockSize, (short) numDataNodes, 0L);
waitForBlockReplication(testFile, dfsClient.getNamenode(), numDataNodes, 20);
Iterable<Block>[] blocksList = cluster.getAllBlockReports(bpid);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java Thu Oct 11 06:14:26 2012
@@ -50,19 +50,6 @@ public class TestModTime {
Random myrand = new Random();
Path hostsFile;
Path excludeFile;
-
- private void writeFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- // create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- (short) repl, blockSize);
- byte[] buffer = new byte[fileSize];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
private void cleanupFile(FileSystem fileSys, Path name) throws IOException {
assertTrue(fileSys.exists(name));
@@ -105,7 +92,8 @@ public class TestModTime {
System.out.println("Creating testdir1 and testdir1/test1.dat.");
Path dir1 = new Path("testdir1");
Path file1 = new Path(dir1, "test1.dat");
- writeFile(fileSys, file1, replicas);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) replicas, seed);
FileStatus stat = fileSys.getFileStatus(file1);
long mtime1 = stat.getModificationTime();
assertTrue(mtime1 != 0);
@@ -120,7 +108,8 @@ public class TestModTime {
//
System.out.println("Creating testdir1/test2.dat.");
Path file2 = new Path(dir1, "test2.dat");
- writeFile(fileSys, file2, replicas);
+ DFSTestUtil.createFile(fileSys, file2, fileSize, fileSize, blockSize,
+ (short) replicas, seed);
stat = fileSys.getFileStatus(file2);
//
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java Thu Oct 11 06:14:26 2012
@@ -83,7 +83,7 @@ public class TestParallelReadUtil {
static class DirectReadWorkerHelper implements ReadWorkerHelper {
@Override
public int read(DFSInputStream dis, byte[] target, int startOff, int len) throws IOException {
- ByteBuffer bb = ByteBuffer.wrap(target);
+ ByteBuffer bb = ByteBuffer.allocateDirect(target.length);
int cnt = 0;
synchronized(dis) {
dis.seek(startOff);
@@ -95,6 +95,8 @@ public class TestParallelReadUtil {
cnt += read;
}
}
+ bb.clear();
+ bb.get(target);
return cnt;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java Thu Oct 11 06:14:26 2012
@@ -41,11 +41,9 @@ public class TestPread {
boolean simulatedStorage = false;
private void writeFile(FileSystem fileSys, Path name) throws IOException {
- // create and write a file that contains three blocks of data
- DataOutputStream stm = fileSys.create(name, true, 4096, (short)1,
- blockSize);
// test empty file open and read
- stm.close();
+ DFSTestUtil.createFile(fileSys, name, 12 * blockSize, 0,
+ blockSize, (short) 1, seed);
FSDataInputStream in = fileSys.open(name);
byte[] buffer = new byte[12 * blockSize];
in.readFully(0, buffer, 0, 0);
@@ -62,11 +60,8 @@ public class TestPread {
assertTrue("Cannot delete file", false);
// now create the real file
- stm = fileSys.create(name, true, 4096, (short)1, blockSize);
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
+ DFSTestUtil.createFile(fileSys, name, 12 * blockSize, 12 * blockSize,
+ blockSize, (short) 1, seed);
}
private void checkAndEraseData(byte[] actual, int from, byte[] expected, String message) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java Thu Oct 11 06:14:26 2012
@@ -26,15 +26,12 @@ import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.util.Iterator;
-import java.util.Random;
import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -61,19 +58,6 @@ public class TestReplication {
private static final int numDatanodes = racks.length;
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestReplication");
-
- private void writeFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- // create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- (short) repl, blockSize);
- byte[] buffer = new byte[fileSize];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
/* check if there are at least two nodes are on the same rack */
private void checkFile(FileSystem fileSys, Path name, int repl)
@@ -222,19 +206,25 @@ public class TestReplication {
FileSystem fileSys = cluster.getFileSystem();
try {
Path file1 = new Path("/smallblocktest.dat");
- writeFile(fileSys, file1, 3);
+ //writeFile(fileSys, file1, 3);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 3, seed);
checkFile(fileSys, file1, 3);
cleanupFile(fileSys, file1);
- writeFile(fileSys, file1, 10);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 10, seed);
checkFile(fileSys, file1, 10);
cleanupFile(fileSys, file1);
- writeFile(fileSys, file1, 4);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 4, seed);
checkFile(fileSys, file1, 4);
cleanupFile(fileSys, file1);
- writeFile(fileSys, file1, 1);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 1, seed);
checkFile(fileSys, file1, 1);
cleanupFile(fileSys, file1);
- writeFile(fileSys, file1, 2);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 2, seed);
checkFile(fileSys, file1, 2);
cleanupFile(fileSys, file1);
} finally {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java Thu Oct 11 06:14:26 2012
@@ -40,16 +40,6 @@ public class TestSeekBug {
static final long seed = 0xDEADBEEFL;
static final int ONEMB = 1 << 20;
- private void writeFile(FileSystem fileSys, Path name) throws IOException {
- // create and write a file that contains 1MB
- DataOutputStream stm = fileSys.create(name);
- byte[] buffer = new byte[ONEMB];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
-
private void checkAndEraseData(byte[] actual, int from, byte[] expected, String message) {
for (int idx = 0; idx < actual.length; idx++) {
assertEquals(message+" byte "+(from+idx)+" differs. expected "+
@@ -132,7 +122,9 @@ public class TestSeekBug {
FileSystem fileSys = cluster.getFileSystem();
try {
Path file1 = new Path("seektest.dat");
- writeFile(fileSys, file1);
+ DFSTestUtil.createFile(fileSys, file1, ONEMB, ONEMB,
+ fileSys.getDefaultBlockSize(file1),
+ fileSys.getDefaultReplication(file1), seed);
seekReadFile(fileSys, file1);
smallReadSeek(fileSys, file1);
cleanupFile(fileSys, file1);
@@ -151,7 +143,9 @@ public class TestSeekBug {
FileSystem fileSys = FileSystem.getLocal(conf);
try {
Path file1 = new Path("build/test/data", "seektest.dat");
- writeFile(fileSys, file1);
+ DFSTestUtil.createFile(fileSys, file1, ONEMB, ONEMB,
+ fileSys.getDefaultBlockSize(file1),
+ fileSys.getDefaultReplication(file1), seed);
seekReadFile(fileSys, file1);
cleanupFile(fileSys, file1);
} finally {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java Thu Oct 11 06:14:26 2012
@@ -115,6 +115,14 @@ public class TestShortCircuitLocalRead {
stm.close();
}
+ private static byte [] arrayFromByteBuffer(ByteBuffer buf) {
+ ByteBuffer alt = buf.duplicate();
+ alt.clear();
+ byte[] arr = new byte[alt.remaining()];
+ alt.get(arr);
+ return arr;
+ }
+
/**
* Verifies that reading a file with the direct read(ByteBuffer) api gives the expected set of bytes.
*/
@@ -122,7 +130,7 @@ public class TestShortCircuitLocalRead {
int readOffset) throws IOException {
HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);
- ByteBuffer actual = ByteBuffer.allocate(expected.length - readOffset);
+ ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
IOUtils.skipFully(stm, readOffset);
@@ -136,7 +144,8 @@ public class TestShortCircuitLocalRead {
// Read across chunk boundary
actual.limit(Math.min(actual.capacity(), nread + 517));
nread += stm.read(actual);
- checkData(actual.array(), readOffset, expected, nread, "A few bytes");
+ checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
+ "A few bytes");
//Now read rest of it
actual.limit(actual.capacity());
while (actual.hasRemaining()) {
@@ -147,7 +156,7 @@ public class TestShortCircuitLocalRead {
}
nread += nbytes;
}
- checkData(actual.array(), readOffset, expected, "Read 3");
+ checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
stm.close();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java Thu Oct 11 06:14:26 2012
@@ -42,18 +42,6 @@ public class TestSmallBlock {
static final int blockSize = 1;
static final int fileSize = 20;
boolean simulatedStorage = false;
-
- private void writeFile(FileSystem fileSys, Path name) throws IOException {
- // create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- (short) 1, blockSize);
- byte[] buffer = new byte[fileSize];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
private void checkAndEraseData(byte[] actual, int from, byte[] expected, String message) {
for (int idx = 0; idx < actual.length; idx++) {
@@ -105,7 +93,8 @@ public class TestSmallBlock {
FileSystem fileSys = cluster.getFileSystem();
try {
Path file1 = new Path("smallblocktest.dat");
- writeFile(fileSys, file1);
+ DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
+ (short) 1, seed);
checkFile(fileSys, file1);
cleanupFile(fileSys, file1);
} finally {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithEncryptedTransfer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithEncryptedTransfer.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithEncryptedTransfer.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithEncryptedTransfer.java Thu Oct 11 06:14:26 2012
@@ -33,17 +33,17 @@ public class TestBalancerWithEncryptedTr
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
}
- @Test
+ @Test(timeout=60000)
public void testEncryptedBalancer0() throws Exception {
new TestBalancer().testBalancer0Internal(conf);
}
- @Test
+ @Test(timeout=60000)
public void testEncryptedBalancer1() throws Exception {
new TestBalancer().testBalancer1Internal(conf);
}
- @Test
+ @Test(timeout=60000)
public void testEncryptedBalancer2() throws Exception {
new TestBalancer().testBalancer2Internal(conf);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java Thu Oct 11 06:14:26 2012
@@ -53,7 +53,7 @@ public class TestRBWBlockInvalidation {
* datanode, namenode should ask to invalidate that corrupted block and
* schedule replication for one more replica for that under replicated block.
*/
- @Test
+ @Test(timeout=60000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
throws IOException, InterruptedException {
Configuration conf = new HdfsConfiguration();