You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ss...@apache.org on 2012/10/16 02:03:59 UTC
svn commit: r1398581 [6/9] - in
/hadoop/common/branches/MR-3902/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-h...
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Tue Oct 16 00:02:55 2012
@@ -159,10 +159,11 @@ class NameNodeRpcServer implements Namen
int handlerCount =
conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY,
DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
- InetSocketAddress socAddr = nn.getRpcServerAddress(conf);
- RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
- ProtobufRpcEngine.class);
- ClientNamenodeProtocolServerSideTranslatorPB
+
+ RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ ClientNamenodeProtocolServerSideTranslatorPB
clientProtocolServerTranslator =
new ClientNamenodeProtocolServerSideTranslatorPB(this);
BlockingService clientNNPbService = ClientNamenodeProtocol.
@@ -199,19 +200,24 @@ class NameNodeRpcServer implements Namen
.newReflectiveBlockingService(haServiceProtocolXlator);
WritableRpcEngine.ensureInitialized();
-
- InetSocketAddress dnSocketAddr = nn.getServiceRpcServerAddress(conf);
- if (dnSocketAddr != null) {
+
+ InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
+ if (serviceRpcAddr != null) {
int serviceHandlerCount =
conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
+ serviceRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(serviceRpcAddr.getHostName())
+ .setPort(serviceRpcAddr.getPort())
+ .setNumHandlers(serviceHandlerCount)
+ .setVerbose(false)
+ .setSecretManager(namesystem.getDelegationTokenSecretManager())
+ .build();
+
// Add all the RPC protocols that the namenode implements
- this.serviceRpcServer =
- RPC.getServer(org.apache.hadoop.hdfs.protocolPB.
- ClientNamenodeProtocolPB.class, clientNNPbService,
- dnSocketAddr.getHostName(), dnSocketAddr.getPort(),
- serviceHandlerCount,
- false, conf, namesystem.getDelegationTokenSecretManager());
DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService,
serviceRpcServer);
DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
@@ -225,18 +231,26 @@ class NameNodeRpcServer implements Namen
DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
getUserMappingService, serviceRpcServer);
- this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress();
+ serviceRPCAddress = serviceRpcServer.getListenerAddress();
nn.setRpcServiceServerAddress(conf, serviceRPCAddress);
} else {
serviceRpcServer = null;
serviceRPCAddress = null;
}
+
+ InetSocketAddress rpcAddr = nn.getRpcServerAddress(conf);
+ clientRpcServer = new RPC.Builder(conf)
+ .setProtocol(
+ org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(rpcAddr.getHostName())
+ .setPort(rpcAddr.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .setSecretManager(namesystem.getDelegationTokenSecretManager())
+ .build();
+
// Add all the RPC protocols that the namenode implements
- this.clientRpcServer = RPC.getServer(
- org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class,
- clientNNPbService, socAddr.getHostName(),
- socAddr.getPort(), handlerCount, false, conf,
- namesystem.getDelegationTokenSecretManager());
DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService,
clientRpcServer);
DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
@@ -254,44 +268,54 @@ class NameNodeRpcServer implements Namen
if (serviceAuthEnabled =
conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
- this.clientRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
- if (this.serviceRpcServer != null) {
- this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ clientRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ if (serviceRpcServer != null) {
+ serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
}
}
// The rpc-server port can be ephemeral... ensure we have the correct info
- this.clientRpcAddress = this.clientRpcServer.getListenerAddress();
+ clientRpcAddress = clientRpcServer.getListenerAddress();
nn.setRpcServerAddress(conf, clientRpcAddress);
- this.minimumDataNodeVersion = conf.get(
+ minimumDataNodeVersion = conf.get(
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY,
DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT);
// Set terse exception whose stack trace won't be logged
- this.clientRpcServer.addTerseExceptions(SafeModeException.class);
+ clientRpcServer.addTerseExceptions(SafeModeException.class);
}
/**
- * Actually start serving requests.
+ * Start client and service RPC servers.
*/
void start() {
- clientRpcServer.start(); //start RPC server
+ clientRpcServer.start();
if (serviceRpcServer != null) {
serviceRpcServer.start();
}
}
/**
- * Wait until the RPC server has shut down.
+ * Wait until the RPC servers have shutdown.
*/
void join() throws InterruptedException {
- this.clientRpcServer.join();
+ clientRpcServer.join();
+ if (serviceRpcServer != null) {
+ serviceRpcServer.join();
+ }
}
-
+
+ /**
+ * Stop client and service RPC servers.
+ */
void stop() {
- if(clientRpcServer != null) clientRpcServer.stop();
- if(serviceRpcServer != null) serviceRpcServer.stop();
+ if (clientRpcServer != null) {
+ clientRpcServer.stop();
+ }
+ if (serviceRpcServer != null) {
+ serviceRpcServer.stop();
+ }
}
InetSocketAddress getServiceRpcAddress() {
@@ -328,8 +352,9 @@ class NameNodeRpcServer implements Namen
namesystem.checkOperation(OperationCategory.UNCHECKED);
verifyRequest(registration);
LOG.info("Error report from " + registration + ": " + msg);
- if(errorCode == FATAL)
+ if (errorCode == FATAL) {
namesystem.releaseBackupNode(registration);
+ }
}
@Override // NamenodeProtocol
@@ -704,6 +729,13 @@ class NameNodeRpcServer implements Namen
namesystem.checkOperation(OperationCategory.UNCHECKED);
namesystem.saveNamespace();
}
+
+ @Override // ClientProtocol
+ public long rollEdits() throws AccessControlException, IOException {
+ namesystem.checkOperation(OperationCategory.JOURNAL);
+ CheckpointSignature sig = namesystem.rollEditLog();
+ return sig.getCurSegmentTxId();
+ }
@Override // ClientProtocol
public void refreshNodes() throws IOException {
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Tue Oct 16 00:02:55 2012
@@ -48,10 +48,10 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -61,6 +61,8 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.znerd.xmlenc.XMLOutputter;
+import com.google.common.base.Preconditions;
+
class NamenodeJspHelper {
static String getSafeModeText(FSNamesystem fsn) {
if (!fsn.isInSafeMode())
@@ -213,6 +215,52 @@ class NamenodeJspHelper {
out.print("</table></div>\n");
}
+
+ /**
+ * Generate an HTML report containing the current status of the HDFS
+ * journals.
+ */
+ void generateJournalReport(JspWriter out, NameNode nn,
+ HttpServletRequest request) throws IOException {
+ FSEditLog log = nn.getFSImage().getEditLog();
+ Preconditions.checkArgument(log != null, "no edit log set in %s", nn);
+
+ out.println("<h3> " + nn.getRole() + " Journal Status: </h3>");
+
+ out.println("<b>Current transaction ID:</b> " +
+ nn.getFSImage().getLastAppliedOrWrittenTxId() + "<br/>");
+
+
+ boolean openForWrite = log.isOpenForWrite();
+
+ out.println("<div class=\"dfstable\">");
+ out.println("<table class=\"storage\" title=\"NameNode Journals\">\n"
+ + "<thead><tr><td><b>Journal Manager</b></td><td><b>State</b></td></tr></thead>");
+ for (JournalAndStream jas : log.getJournals()) {
+ out.print("<tr>");
+ out.print("<td>" + jas.getManager());
+ if (jas.isRequired()) {
+ out.print(" [required]");
+ }
+ out.print("</td><td>");
+
+ if (jas.isDisabled()) {
+ out.print("<span class=\"failed\">Failed</span>");
+ } else if (openForWrite) {
+ EditLogOutputStream elos = jas.getCurrentStream();
+ if (elos != null) {
+ out.println(elos.generateHtmlReport());
+ } else {
+ out.println("not currently writing");
+ }
+ } else {
+ out.println("open for read");
+ }
+ out.println("</td></tr>");
+ }
+
+ out.println("</table></div>");
+ }
void generateHealthReport(JspWriter out, NameNode nn,
HttpServletRequest request) throws IOException {
@@ -395,7 +443,7 @@ class NamenodeJspHelper {
nodeToRedirect = nn.getHttpAddress().getHostName();
redirectPort = nn.getHttpAddress().getPort();
}
- String addr = NetUtils.getHostPortString(nn.getNameNodeAddress());
+ String addr = nn.getNameNodeAddressHostPortString();
String fqdn = InetAddress.getByName(nodeToRedirect).getCanonicalHostName();
redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":" + redirectPort
+ "/browseDirectory.jsp?namenodeInfoPort="
@@ -566,8 +614,9 @@ class NamenodeJspHelper {
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
dm.fetchDatanodes(live, dead, true);
- InetSocketAddress nnSocketAddress = (InetSocketAddress) context
- .getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY);
+ InetSocketAddress nnSocketAddress =
+ (InetSocketAddress)context.getAttribute(
+ NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY);
String nnaddr = nnSocketAddress.getAddress().getHostAddress() + ":"
+ nnSocketAddress.getPort();
@@ -785,7 +834,7 @@ class NamenodeJspHelper {
doc.endTag();
doc.startTag("replication");
- doc.pcdata(""+inode.getReplication());
+ doc.pcdata(""+inode.getBlockReplication());
doc.endTag();
doc.startTag("disk_space_consumed");
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java Tue Oct 16 00:02:55 2012
@@ -20,13 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Oct 16 00:02:55 2012
@@ -58,6 +58,8 @@ import org.apache.hadoop.hdfs.server.com
import static org.apache.hadoop.util.ExitUtil.terminate;
+import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -76,6 +78,7 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
/**********************************************************
@@ -120,6 +123,8 @@ public class SecondaryNameNode implement
private CheckpointConf checkpointConf;
private FSNamesystem namesystem;
+ private Thread checkpointThread;
+
@Override
public String toString() {
@@ -138,6 +143,11 @@ public class SecondaryNameNode implement
FSImage getFSImage() {
return checkpointImage;
}
+
+ @VisibleForTesting
+ FSNamesystem getFSNamesystem() {
+ return namesystem;
+ }
@VisibleForTesting
void setFSImage(CheckpointStorage image) {
@@ -270,6 +280,15 @@ public class SecondaryNameNode implement
*/
public void shutdown() {
shouldRun = false;
+ if (checkpointThread != null) {
+ checkpointThread.interrupt();
+ try {
+ checkpointThread.join(10000);
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted waiting to join on checkpointer thread");
+ Thread.currentThread().interrupt(); // maintain status
+ }
+ }
try {
if (infoServer != null) infoServer.stop();
} catch (Exception e) {
@@ -473,10 +492,6 @@ public class SecondaryNameNode implement
LOG.warn("Checkpoint done. New Image Size: "
+ dstStorage.getFsImageName(txid).length());
- // Since we've successfully checkpointed, we can remove some old
- // image files
- checkpointImage.purgeOldStorage();
-
return loadImage;
}
@@ -583,12 +598,20 @@ public class SecondaryNameNode implement
terminate(ret);
}
- // Create a never ending deamon
- Daemon checkpointThread = new Daemon(secondary);
- checkpointThread.start();
+ secondary.startCheckpointThread();
}
+ public void startCheckpointThread() {
+ Preconditions.checkState(checkpointThread == null,
+ "Should not already have a thread");
+ Preconditions.checkState(shouldRun, "shouldRun should be true");
+
+ checkpointThread = new Daemon(this);
+ checkpointThread.start();
+ }
+
+
/**
* Container for parsed command-line options.
*/
@@ -703,6 +726,34 @@ public class SecondaryNameNode implement
}
static class CheckpointStorage extends FSImage {
+
+ private static class CheckpointLogPurger implements LogsPurgeable {
+
+ private NNStorage storage;
+ private StoragePurger purger
+ = new NNStorageRetentionManager.DeletionStoragePurger();
+
+ public CheckpointLogPurger(NNStorage storage) {
+ this.storage = storage;
+ }
+
+ @Override
+ public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException {
+ Iterator<StorageDirectory> iter = storage.dirIterator();
+ while (iter.hasNext()) {
+ StorageDirectory dir = iter.next();
+ List<EditLogFile> editFiles = FileJournalManager.matchEditLogs(
+ dir.getCurrentDir());
+ for (EditLogFile f : editFiles) {
+ if (f.getLastTxId() < minTxIdToKeep) {
+ purger.purgeLog(f);
+ }
+ }
+ }
+ }
+
+ }
+
/**
* Construct a checkpoint image.
* @param conf Node configuration.
@@ -719,6 +770,11 @@ public class SecondaryNameNode implement
// we shouldn't have any editLog instance. Setting to null
// makes sure we don't accidentally depend on it.
editLog = null;
+
+ // Replace the archival manager with one that can actually work on the
+ // 2NN's edits storage.
+ this.archivalManager = new NNStorageRetentionManager(conf, storage,
+ new CheckpointLogPurger(storage));
}
/**
@@ -815,6 +871,7 @@ public class SecondaryNameNode implement
}
Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
+ // The following has the side effect of purging old fsimages/edit logs.
dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
dstStorage.writeAll();
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java Tue Oct 16 00:02:55 2012
@@ -75,7 +75,7 @@ class SerialNumberManager {
return t;
}
- /** {@inheritDoc} */
+ @Override
public String toString() {
return "max=" + max + ",\n t2i=" + t2i + ",\n i2t=" + i2t;
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Tue Oct 16 00:02:55 2012
@@ -32,9 +32,12 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -43,6 +46,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.MD5Hash;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -54,6 +58,8 @@ public class TransferFsImage {
public final static String CONTENT_LENGTH = "Content-Length";
public final static String MD5_HEADER = "X-MD5-Digest";
+ @VisibleForTesting
+ static int timeout = 0;
private static final Log LOG = LogFactory.getLog(TransferFsImage.class);
@@ -222,6 +228,18 @@ public class TransferFsImage {
HttpURLConnection connection = (HttpURLConnection)
SecurityUtil.openSecureHttpConnection(url);
+ if (timeout <= 0) {
+ // Set the ping interval as timeout
+ Configuration conf = new HdfsConfiguration();
+ timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY,
+ DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT);
+ }
+
+ if (timeout > 0) {
+ connection.setConnectTimeout(timeout);
+ connection.setReadTimeout(timeout);
+ }
+
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
throw new HttpGetFailedException(
"Image transfer servlet at " + url +
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java Tue Oct 16 00:02:55 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import org.apache.commons.logging.Log;
@@ -172,11 +173,20 @@ public class EditLogTailer {
Preconditions.checkState(tailerThread == null ||
!tailerThread.isAlive(),
"Tailer thread should not be running once failover starts");
- try {
- doTailEdits();
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
+ // Important to do tailing as the login user, in case the shared
+ // edits storage is implemented by a JournalManager that depends
+ // on security credentials to access the logs (eg QuorumJournalManager).
+ SecurityUtil.doAsLoginUser(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ doTailEdits();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ return null;
+ }
+ });
}
@VisibleForTesting
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Tue Oct 16 00:02:55 2012
@@ -21,9 +21,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.EnumSet;
import javax.servlet.ServletContext;
@@ -92,6 +94,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -116,9 +119,20 @@ public class NamenodeWebHdfsMethods {
return REMOTE_ADDRESS.get();
}
- /** Set the remote client address. */
- static void setRemoteAddress(String remoteAddress) {
- REMOTE_ADDRESS.set(remoteAddress);
+ public static InetAddress getRemoteIp() {
+ try {
+ return InetAddress.getByName(getRemoteAddress());
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Returns true if a WebHdfs request is in progress. Akin to
+ * {@link Server#isRpcInvocation()}.
+ */
+ public static boolean isWebHdfsInvocation() {
+ return getRemoteAddress() != null;
}
private @Context ServletContext context;
@@ -150,8 +164,9 @@ public class NamenodeWebHdfsMethods {
final DatanodeDescriptor clientNode = bm.getDatanodeManager(
).getDatanodeByHost(getRemoteAddress());
if (clientNode != null) {
- final DatanodeDescriptor[] datanodes = bm.getBlockPlacementPolicy(
- ).chooseTarget(path, 1, clientNode, null, blocksize);
+ final DatanodeDescriptor[] datanodes = bm.getBlockPlacementPolicy()
+ .chooseTarget(path, 1, clientNode,
+ new ArrayList<DatanodeDescriptor>(), false, null, blocksize);
if (datanodes.length > 0) {
return datanodes[0];
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java Tue Oct 16 00:02:55 2012
@@ -88,4 +88,13 @@ public class DatanodeRegistration extend
+ ", storageInfo=" + storageInfo
+ ")";
}
+
+ @Override
+ public boolean equals(Object to) {
+ return super.equals(to);
+ }
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLog.java Tue Oct 16 00:02:55 2012
@@ -17,18 +17,15 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.io.Writable;
import com.google.common.base.Function;
import com.google.common.collect.ComparisonChain;
-public class RemoteEditLog implements Writable, Comparable<RemoteEditLog> {
+public class RemoteEditLog implements Comparable<RemoteEditLog> {
private long startTxId = HdfsConstants.INVALID_TXID;
private long endTxId = HdfsConstants.INVALID_TXID;
+ private boolean isInProgress = false;
public RemoteEditLog() {
}
@@ -36,6 +33,13 @@ public class RemoteEditLog implements Wr
public RemoteEditLog(long startTxId, long endTxId) {
this.startTxId = startTxId;
this.endTxId = endTxId;
+ this.isInProgress = (endTxId == HdfsConstants.INVALID_TXID);
+ }
+
+ public RemoteEditLog(long startTxId, long endTxId, boolean inProgress) {
+ this.startTxId = startTxId;
+ this.endTxId = endTxId;
+ this.isInProgress = inProgress;
}
public long getStartTxId() {
@@ -45,22 +49,18 @@ public class RemoteEditLog implements Wr
public long getEndTxId() {
return endTxId;
}
-
- @Override
- public String toString() {
- return "[" + startTxId + "," + endTxId + "]";
- }
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(startTxId);
- out.writeLong(endTxId);
+ public boolean isInProgress() {
+ return isInProgress;
}
@Override
- public void readFields(DataInput in) throws IOException {
- startTxId = in.readLong();
- endTxId = in.readLong();
+ public String toString() {
+ if (!isInProgress) {
+ return "[" + startTxId + "," + endTxId + "]";
+ } else {
+ return "[" + startTxId + "-? (in-progress)]";
+ }
}
@Override
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java Tue Oct 16 00:02:55 2012
@@ -40,8 +40,8 @@ public class RemoteEditLogManifest {
/**
- * Check that the logs are contiguous and non-overlapping
- * sequences of transactions, in sorted order
+ * Check that the logs are non-overlapping sequences of transactions,
+ * in sorted order. They do not need to be contiguous.
* @throws IllegalStateException if incorrect
*/
private void checkState() {
@@ -50,8 +50,10 @@ public class RemoteEditLogManifest {
RemoteEditLog prev = null;
for (RemoteEditLog log : logs) {
if (prev != null) {
- if (log.getStartTxId() != prev.getEndTxId() + 1) {
- throw new IllegalStateException("Invalid log manifest:" + this);
+ if (log.getStartTxId() <= prev.getEndTxId()) {
+ throw new IllegalStateException(
+ "Invalid log manifest (log " + log + " overlaps " + prev + ")\n"
+ + this);
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Tue Oct 16 00:02:55 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.tools;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -53,6 +54,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.util.StringUtils;
@@ -80,7 +82,7 @@ public class DFSAdmin extends FsShell {
super(fs.getConf());
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not a distributed file system");
+ " is not an HDFS file system");
}
this.dfs = (DistributedFileSystem)fs;
}
@@ -284,7 +286,7 @@ public class DFSAdmin extends FsShell {
FileSystem fs = getFS();
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
- " is not a distributed file system");
+ " is not an HDFS file system");
}
return (DistributedFileSystem)fs;
}
@@ -420,6 +422,14 @@ public class DFSAdmin extends FsShell {
return exitCode;
}
+ public int rollEdits() throws IOException {
+ DistributedFileSystem dfs = getDFS();
+ long txid = dfs.rollEdits();
+ System.out.println("Successfully rolled edit logs.");
+ System.out.println("New segment starts at txid " + txid);
+ return 0;
+ }
+
/**
* Command to enable/disable/check restoring of failed storage replicas in the namenode.
* Usage: java DFSAdmin -restoreFailedStorage true|false|check
@@ -503,11 +513,17 @@ public class DFSAdmin extends FsShell {
* @return an exit code indicating success or failure.
* @throws IOException
*/
- public int fetchImage(String[] argv, int idx) throws IOException {
- String infoServer = DFSUtil.getInfoServer(
+ public int fetchImage(final String[] argv, final int idx) throws IOException {
+ final String infoServer = DFSUtil.getInfoServer(
HAUtil.getAddressOfActive(getDFS()), getConf(), false);
- TransferFsImage.downloadMostRecentImageToDirectory(infoServer,
- new File(argv[idx]));
+ SecurityUtil.doAsCurrentUser(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ TransferFsImage.downloadMostRecentImageToDirectory(infoServer,
+ new File(argv[idx]));
+ return null;
+ }
+ });
return 0;
}
@@ -516,6 +532,7 @@ public class DFSAdmin extends FsShell {
"The full syntax is: \n\n" +
"hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
"\t[-saveNamespace]\n" +
+ "\t[-rollEdits]\n" +
"\t[-restoreFailedStorage true|false|check]\n" +
"\t[-refreshNodes]\n" +
"\t[" + SetQuotaCommand.USAGE + "]\n" +
@@ -548,6 +565,10 @@ public class DFSAdmin extends FsShell {
"Save current namespace into storage directories and reset edits log.\n" +
"\t\tRequires superuser permissions and safe mode.\n";
+ String rollEdits = "-rollEdits:\t" +
+ "Rolls the edit log.\n" +
+ "\t\tRequires superuser permissions.\n";
+
String restoreFailedStorage = "-restoreFailedStorage:\t" +
"Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" +
"\t\tRequires superuser permissions.\n";
@@ -625,6 +646,8 @@ public class DFSAdmin extends FsShell {
System.out.println(safemode);
} else if ("saveNamespace".equals(cmd)) {
System.out.println(saveNamespace);
+ } else if ("rollEdits".equals(cmd)) {
+ System.out.println(rollEdits);
} else if ("restoreFailedStorage".equals(cmd)) {
System.out.println(restoreFailedStorage);
} else if ("refreshNodes".equals(cmd)) {
@@ -664,6 +687,7 @@ public class DFSAdmin extends FsShell {
System.out.println(report);
System.out.println(safemode);
System.out.println(saveNamespace);
+ System.out.println(rollEdits);
System.out.println(restoreFailedStorage);
System.out.println(refreshNodes);
System.out.println(finalizeUpgrade);
@@ -859,6 +883,9 @@ public class DFSAdmin extends FsShell {
} else if ("-saveNamespace".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-saveNamespace]");
+ } else if ("-rollEdits".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-rollEdits]");
} else if ("-restoreFailedStorage".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-restoreFailedStorage true|false|check ]");
@@ -913,6 +940,7 @@ public class DFSAdmin extends FsShell {
System.err.println(" [-report]");
System.err.println(" [-safemode enter | leave | get | wait]");
System.err.println(" [-saveNamespace]");
+ System.err.println(" [-rollEdits]");
System.err.println(" [-restoreFailedStorage true|false|check]");
System.err.println(" [-refreshNodes]");
System.err.println(" [-finalizeUpgrade]");
@@ -970,6 +998,11 @@ public class DFSAdmin extends FsShell {
printUsage(cmd);
return exitCode;
}
+ } else if ("-rollEdits".equals(cmd)) {
+ if (argv.length != 1) {
+ printUsage(cmd);
+ return exitCode;
+ }
} else if ("-restoreFailedStorage".equals(cmd)) {
if (argv.length != 2) {
printUsage(cmd);
@@ -1048,6 +1081,8 @@ public class DFSAdmin extends FsShell {
setSafeMode(argv, i);
} else if ("-saveNamespace".equals(cmd)) {
exitCode = saveNamespace();
+ } else if ("-rollEdits".equals(cmd)) {
+ exitCode = rollEdits();
} else if ("-restoreFailedStorage".equals(cmd)) {
exitCode = restoreFaileStorage(argv[i]);
} else if ("-refreshNodes".equals(cmd)) {
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java Tue Oct 16 00:02:55 2012
@@ -56,7 +56,7 @@ public class BinaryEditsVisitor implemen
@Override
public void close(Throwable error) throws IOException {
elfos.setReadyToFlush();
- elfos.flushAndSync();
+ elfos.flushAndSync(true);
elfos.close();
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java Tue Oct 16 00:02:55 2012
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Tue Oct 16 00:02:55 2012
@@ -31,13 +31,11 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
-import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.xml.sax.helpers.AttributesImpl;
/**
* ImageLoaderCurrent processes Hadoop FSImage files and walks over
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/CyclicIteration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/CyclicIteration.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/CyclicIteration.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/CyclicIteration.java Tue Oct 16 00:02:55 2012
@@ -53,7 +53,6 @@ public class CyclicIteration<K, V> imple
}
}
- /** {@inheritDoc} */
@Override
public Iterator<Map.Entry<K, V>> iterator() {
return new CyclicIterator();
@@ -89,13 +88,11 @@ public class CyclicIteration<K, V> imple
return i.next();
}
- /** {@inheritDoc} */
@Override
public boolean hasNext() {
return hasnext;
}
- /** {@inheritDoc} */
@Override
public Map.Entry<K, V> next() {
if (!hasnext) {
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java Tue Oct 16 00:02:55 2012
@@ -57,7 +57,9 @@ public class PersistentLongFile {
}
public void set(long newVal) throws IOException {
- writeFile(file, newVal);
+ if (value != newVal || !loaded) {
+ writeFile(file, newVal);
+ }
value = newVal;
loaded = true;
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Tue Oct 16 00:02:55 2012
@@ -54,7 +54,6 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hdfs.ByteRangeInputStream;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -89,6 +88,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -181,7 +181,14 @@ public class WebHdfsFileSystem extends F
throw new IllegalArgumentException(e);
}
this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
- this.retryPolicy = NameNodeProxies.getDefaultRetryPolicy(conf);
+ this.retryPolicy =
+ RetryUtils.getDefaultRetryPolicy(
+ conf,
+ DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
+ DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
+ DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
+ DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
+ SafeModeException.class);
this.workingDir = getHomeDirectory();
if (UserGroupInformation.isSecurityEnabled()) {
Propchange: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c Tue Oct 16 00:02:55 2012
@@ -67,6 +67,25 @@ static const struct ExceptionInfo gExcep
};
+void getExceptionInfo(const char *excName, int noPrintFlags,
+ int *excErrno, int *shouldPrint)
+{
+ int i;
+
+ for (i = 0; i < EXCEPTION_INFO_LEN; i++) {
+ if (strstr(gExceptionInfo[i].name, excName)) {
+ break;
+ }
+ }
+ if (i < EXCEPTION_INFO_LEN) {
+ *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags);
+ *excErrno = gExceptionInfo[i].excErrno;
+ } else {
+ *shouldPrint = 1;
+ *excErrno = EINTERNAL;
+ }
+}
+
int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags,
const char *fmt, va_list ap)
{
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.h Tue Oct 16 00:02:55 2012
@@ -65,6 +65,21 @@
#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10
/**
+ * Get information about an exception.
+ *
+ * @param excName The Exception name.
+ * This is a Java class name in JNI format.
+ * @param noPrintFlags Flags which determine which exceptions we should NOT
+ * print.
+ * @param excErrno (out param) The POSIX error number associated with the
+ * exception.
+ * @param shouldPrint (out param) Nonzero if we should print this exception,
+ * based on the noPrintFlags and its name.
+ */
+void getExceptionInfo(const char *excName, int noPrintFlags,
+ int *excErrno, int *shouldPrint);
+
+/**
* Print out information about an exception and free it.
*
* @param env The JNI environment
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Tue Oct 16 00:02:55 2012
@@ -276,6 +276,13 @@ message SaveNamespaceRequestProto { // n
message SaveNamespaceResponseProto { // void response
}
+message RollEditsRequestProto { // no parameters
+}
+
+message RollEditsResponseProto { // response
+ required uint64 newSegmentTxId = 1;
+}
+
message RestoreFailedStorageRequestProto {
required string arg = 1;
}
@@ -472,6 +479,8 @@ service ClientNamenodeProtocol {
returns(SetSafeModeResponseProto);
rpc saveNamespace(SaveNamespaceRequestProto)
returns(SaveNamespaceResponseProto);
+ rpc rollEdits(RollEditsRequestProto)
+ returns(RollEditsResponseProto);
rpc restoreFailedStorage(RestoreFailedStorageRequestProto)
returns(RestoreFailedStorageResponseProto);
rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto Tue Oct 16 00:02:55 2012
@@ -54,12 +54,7 @@ message OpReadBlockProto {
message ChecksumProto {
- enum ChecksumType {
- NULL = 0;
- CRC32 = 1;
- CRC32C = 2;
- }
- required ChecksumType type = 1;
+ required ChecksumTypeProto type = 1;
required uint32 bytesPerChecksum = 2;
}
@@ -185,4 +180,5 @@ message OpBlockChecksumResponseProto {
required uint32 bytesPerCrc = 1;
required uint64 crcPerBlock = 2;
required bytes md5 = 3;
+ optional ChecksumTypeProto crcType = 4 [default = CRC32];
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Tue Oct 16 00:02:55 2012
@@ -302,6 +302,7 @@ message BlocksWithLocationsProto {
message RemoteEditLogProto {
required uint64 startTxId = 1; // Starting available edit log transaction
required uint64 endTxId = 2; // Ending available edit log transaction
+ optional bool isInProgress = 3 [default = false];
}
/**
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Tue Oct 16 00:02:55 2012
@@ -41,11 +41,34 @@
</property>
<property>
+ <name>dfs.namenode.rpc-address</name>
+ <value></value>
+ <description>
+ RPC address that handles all clients requests. In the case of HA/Federation where multiple namenodes exist,
+ the name service id is added to the name e.g. dfs.namenode.rpc-address.ns1
+ dfs.namenode.rpc-address.EXAMPLENAMESERVICE
+ The value of this property will take the form of hdfs://nn-host1:rpc-port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value></value>
+ <description>
+ RPC address for HDFS Services communication. BackupNode, Datanodes and all other services should be
+ connecting to this address if it is configured. In the case of HA/Federation where multiple namenodes exist,
+ the name service id is added to the name e.g. dfs.namenode.servicerpc-address.ns1
+ dfs.namenode.rpc-address.EXAMPLENAMESERVICE
+ The value of this property will take the form of hdfs://nn-host1:rpc-port.
+ If the value of this property is unset the value of dfs.namenode.rpc-address will be used as the default.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.secondary.http-address</name>
<value>0.0.0.0:50090</value>
<description>
The secondary namenode http server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -54,7 +77,6 @@
<value>0.0.0.0:50010</value>
<description>
The datanode server address and port for data transfer.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -63,7 +85,6 @@
<value>0.0.0.0:50075</value>
<description>
The datanode http server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -72,7 +93,6 @@
<value>0.0.0.0:50020</value>
<description>
The datanode ipc server address and port.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -87,7 +107,6 @@
<value>0.0.0.0:50070</value>
<description>
The address and the base port where the dfs namenode web ui will listen on.
- If the port is 0 then the server will start on a free port.
</description>
</property>
@@ -241,6 +260,11 @@
</property>
<property>
+ <name>dfs.namenode.edits.journal-plugin.qjournal</name>
+ <value>org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager</value>
+</property>
+
+<property>
<name>dfs.permissions.enabled</name>
<value>true</value>
<description>
@@ -961,6 +985,60 @@
</property>
<property>
+ <name>dfs.namenode.check.stale.datanode</name>
+ <value>false</value>
+ <description>
+ Indicate whether or not to check "stale" datanodes whose
+ heartbeat messages have not been received by the namenode
+ for more than a specified time interval. If this configuration
+ parameter is set as true, the system will keep track
+ of the number of stale datanodes. The stale datanodes will be
+ moved to the end of the node list returned for reading. See
+ dfs.namenode.avoid.write.stale.datanode for details on how this
+ affects writes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.avoid.write.stale.datanode</name>
+ <value>false</value>
+ <description>
+ Indicate whether or not to avoid writing to "stale" datanodes whose
+ heartbeat messages have not been received by the namenode
+ for more than a specified time interval. If this configuration
+ parameter and dfs.namenode.check.stale.datanode are both set as true,
+ the writing will avoid using stale datanodes unless a high number
+ of datanodes are marked as stale. See
+ dfs.namenode.write.stale.datanode.ratio for details.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.stale.datanode.interval</name>
+ <value>30000</value>
+ <description>
+ Default time interval for marking a datanode as "stale", i.e., if
+ the namenode has not received heartbeat msg from a datanode for
+ more than this time interval, the datanode will be marked and treated
+ as "stale" by default. The stale interval cannot be too small since
+ otherwise this may cause too frequent change of stale states.
+ We thus set a minimum stale interval value (the default value is 3 times
+ of heartbeat interval) and guarantee that the stale interval cannot be less
+ than the minimum value.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.write.stale.datanode.ratio</name>
+ <value>0.5f</value>
+ <description>
+ When the ratio of number stale datanodes to total datanodes marked
+ is greater than this ratio, stop avoiding writing to stale nodes so
+ as to prevent causing hotspots.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.invalidate.work.pct.per.iteration</name>
<value>0.32f</value>
<description>
@@ -1075,4 +1153,21 @@
</description>
</property>
+<property>
+ <name>dfs.journalnode.rpc-address</name>
+ <value>0.0.0.0:8485</value>
+ <description>
+ The JournalNode RPC server address and port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.http-address</name>
+ <value>0.0.0.0:8480</value>
+ <description>
+ The address and port the JournalNode web UI listens on.
+ If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
</configuration>
Propchange: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1363593-1396941
Propchange: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp Tue Oct 16 00:02:55 2012
@@ -34,8 +34,7 @@
HAServiceState nnHAState = nn.getServiceState();
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
String namenodeRole = nn.getRole().toString();
- String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":"
- + nn.getNameNodeAddress().getPort();
+ String namenodeLabel = nn.getNameNodeAddressHostPortString();
Collection<FSNamesystem.CorruptFileBlockInfo> corruptFileBlocks =
fsn.listCorruptFileBlocks("/", null);
int corruptFileCount = corruptFileBlocks.size();
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp Tue Oct 16 00:02:55 2012
@@ -34,7 +34,7 @@
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
String namenodeRole = nn.getRole().toString();
String namenodeState = nnHAState.toString();
- String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+ String namenodeLabel = nn.getNameNodeAddressHostPortString();
%>
<!DOCTYPE html>
@@ -60,8 +60,10 @@
<%= NamenodeJspHelper.getCorruptFilesWarning(fsn)%>
<% healthjsp.generateHealthReport(out, nn, request); %>
-<hr>
+<% healthjsp.generateJournalReport(out, nn, request); %>
+<hr/>
<% healthjsp.generateConfReport(out, nn, request); %>
+<hr>
<%
out.println(ServletUtil.htmlFooter());
%>
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp Tue Oct 16 00:02:55 2012
@@ -33,7 +33,7 @@ String namenodeRole = nn.getRole().toStr
FSNamesystem fsn = nn.getNamesystem();
HAServiceState nnHAState = nn.getServiceState();
boolean isActive = (nnHAState == HAServiceState.ACTIVE);
-String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+String namenodeLabel = nn.getNameNodeAddressHostPortString();
%>
<!DOCTYPE html>
Propchange: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java Tue Oct 16 00:02:55 2012
@@ -271,7 +271,6 @@ public class DataTransferTestUtil {
}
}
- /** {@inheritDoc} */
@Override
public String toString() {
return error + " " + super.toString();
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/PipelinesTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/PipelinesTestUtil.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/PipelinesTestUtil.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/PipelinesTestUtil.java Tue Oct 16 00:02:55 2012
@@ -51,9 +51,6 @@ public class PipelinesTestUtil extends D
this.name = name;
}
- /**
- * {@inheritDoc}
- */
@Override
public void run(NodeBytes nb) throws IOException {
synchronized (rcv) {
Propchange: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java Tue Oct 16 00:02:55 2012
@@ -17,18 +17,16 @@
*/
package org.apache.hadoop.fs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.*;
public class TestGlobPaths {
@@ -49,26 +47,377 @@ public class TestGlobPaths {
static private MiniDFSCluster dfsCluster;
static private FileSystem fs;
static final private int NUM_OF_PATHS = 4;
- static final String USER_DIR = "/user/"+System.getProperty("user.name");
+ static private String USER_DIR;
private Path[] path = new Path[NUM_OF_PATHS];
- @Before
- public void setUp() throws Exception {
- try {
- Configuration conf = new HdfsConfiguration();
- dfsCluster = new MiniDFSCluster.Builder(conf).build();
- fs = FileSystem.get(conf);
- } catch (IOException e) {
- e.printStackTrace();
- }
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ dfsCluster = new MiniDFSCluster.Builder(conf).build();
+ fs = FileSystem.get(conf);
+ USER_DIR = fs.getHomeDirectory().toUri().getPath().toString();
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
if(dfsCluster!=null) {
dfsCluster.shutdown();
}
}
+
+ @Test
+ public void testMultiGlob() throws IOException {
+ FileStatus[] status;
+ /*
+ * /dir1/subdir1
+ * /dir1/subdir1/f1
+ * /dir1/subdir1/f2
+ * /dir1/subdir2/f1
+ * /dir2/subdir1
+ * /dir2/subdir2
+ * /dir2/subdir2/f1
+ * /dir3/f1
+ * /dir3/f1
+ * /dir3/f2(dir)
+ * /dir3/subdir2(file)
+ * /dir3/subdir3
+ * /dir3/subdir3/f1
+ * /dir3/subdir3/f1/f1
+ * /dir3/subdir3/f3
+ * /dir4
+ */
+
+ Path d1 = new Path(USER_DIR, "dir1");
+ Path d11 = new Path(d1, "subdir1");
+ Path d12 = new Path(d1, "subdir2");
+
+ Path f111 = new Path(d11, "f1");
+ fs.createNewFile(f111);
+ Path f112 = new Path(d11, "f2");
+ fs.createNewFile(f112);
+ Path f121 = new Path(d12, "f1");
+ fs.createNewFile(f121);
+
+ Path d2 = new Path(USER_DIR, "dir2");
+ Path d21 = new Path(d2, "subdir1");
+ fs.mkdirs(d21);
+ Path d22 = new Path(d2, "subdir2");
+ Path f221 = new Path(d22, "f1");
+ fs.createNewFile(f221);
+
+ Path d3 = new Path(USER_DIR, "dir3");
+ Path f31 = new Path(d3, "f1");
+ fs.createNewFile(f31);
+ Path d32 = new Path(d3, "f2");
+ fs.mkdirs(d32);
+ Path f32 = new Path(d3, "subdir2"); // fake as a subdir!
+ fs.createNewFile(f32);
+ Path d33 = new Path(d3, "subdir3");
+ Path f333 = new Path(d33, "f3");
+ fs.createNewFile(f333);
+ Path d331 = new Path(d33, "f1");
+ Path f3311 = new Path(d331, "f1");
+ fs.createNewFile(f3311);
+ Path d4 = new Path(USER_DIR, "dir4");
+ fs.mkdirs(d4);
+
+ /*
+ * basic
+ */
+ Path root = new Path(USER_DIR);
+ status = fs.globStatus(root);
+ checkStatus(status, root);
+
+ status = fs.globStatus(new Path(USER_DIR, "x"));
+ assertNull(status);
+
+ status = fs.globStatus(new Path("x"));
+ assertNull(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "x/x"));
+ assertNull(status);
+
+ status = fs.globStatus(new Path("x/x"));
+ assertNull(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "*"));
+ checkStatus(status, d1, d2, d3, d4);
+
+ status = fs.globStatus(new Path("*"));
+ checkStatus(status, d1, d2, d3, d4);
+
+ status = fs.globStatus(new Path(USER_DIR, "*/x"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("*/x"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "x/*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("x/*"));
+ checkStatus(status);
+
+ // make sure full pattern is scanned instead of bailing early with undef
+ status = fs.globStatus(new Path(USER_DIR, "x/x/x/*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("x/x/x/*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "*/*"));
+ checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
+
+ status = fs.globStatus(new Path("*/*"));
+ checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
+
+ /*
+ * one level deep
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/*"));
+ checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
+
+ status = fs.globStatus(new Path("dir*/*"));
+ checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*"));
+ checkStatus(status, d11, d12, d21, d22, f32, d33);
+
+ status = fs.globStatus(new Path("dir*/subdir*"));
+ checkStatus(status, d11, d12, d21, d22, f32, d33);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/f*"));
+ checkStatus(status, f31, d32);
+
+ status = fs.globStatus(new Path("dir*/f*"));
+ checkStatus(status, f31, d32);
+
+ /*
+ * subdir1 globs
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1"));
+ checkStatus(status, d11, d21);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/*"));
+ checkStatus(status, f111, f112);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/*/*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/x"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/x*"));
+ checkStatus(status);
+
+ /*
+ * subdir2 globs
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2"));
+ checkStatus(status, d12, d22, f32);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2/*"));
+ checkStatus(status, f121, f221);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2/*/*"));
+ checkStatus(status);
+
+ /*
+ * subdir3 globs
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3"));
+ checkStatus(status, d33);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*"));
+ checkStatus(status, d331, f333);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*/*"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*/*/*"));
+ checkStatus(status);
+
+ /*
+ * file1 single dir globs
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1"));
+ checkStatus(status, f111);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1*"));
+ checkStatus(status, f111);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1/*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1*/*"));
+ checkStatus(status);
+
+ /*
+ * file1 multi-dir globs
+ */
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1"));
+ checkStatus(status, f111, f121, f221, d331);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*"));
+ checkStatus(status, f111, f121, f221, d331);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1/*"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/x"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*/*"));
+ checkStatus(status);
+
+ /*
+ * file glob multiple files
+ */
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*"));
+ checkStatus(status, d11, d12, d21, d22, f32, d33);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*"));
+ checkStatus(status, f111, f112, f121, f221, d331, f333);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f*"));
+ checkStatus(status, f111, f112, f121, f221, d331, f333);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f*/*"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*/f1"));
+ checkStatus(status, f3311);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*/*"));
+ checkStatus(status, f3311);
+
+
+ // doesn't exist
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f3"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f3*"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("{x}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("{x,y}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("dir*/{x,y}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("dir*/{f1,y}"));
+ checkStatus(status, f31);
+
+ status = fs.globStatus(new Path("{x,y}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("/{x/x,y/y}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("{x/x,y/y}"));
+ checkStatus(status);
+
+ status = fs.globStatus(new Path(Path.CUR_DIR));
+ checkStatus(status, new Path(USER_DIR));
+
+ status = fs.globStatus(new Path(USER_DIR+"{/dir1}"));
+ checkStatus(status, d1);
+
+ status = fs.globStatus(new Path(USER_DIR+"{/dir*}"));
+ checkStatus(status, d1, d2, d3, d4);
+
+ /*
+ * true filter
+ */
+
+ PathFilter trueFilter = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return true;
+ }
+ };
+
+ status = fs.globStatus(new Path(Path.SEPARATOR), trueFilter);
+ checkStatus(status, new Path(Path.SEPARATOR));
+
+ status = fs.globStatus(new Path(Path.CUR_DIR), trueFilter);
+ checkStatus(status, new Path(USER_DIR));
+
+ status = fs.globStatus(d1, trueFilter);
+ checkStatus(status, d1);
+
+ status = fs.globStatus(new Path(USER_DIR), trueFilter);
+ checkStatus(status, new Path(USER_DIR));
+
+ status = fs.globStatus(new Path(USER_DIR, "*"), trueFilter);
+ checkStatus(status, d1, d2, d3, d4);
+
+ status = fs.globStatus(new Path("/x/*"), trueFilter);
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("/x"), trueFilter);
+ assertNull(status);
+
+ status = fs.globStatus(new Path("/x/x"), trueFilter);
+ assertNull(status);
+
+ /*
+ * false filter
+ */
+ PathFilter falseFilter = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return false;
+ }
+ };
+
+ status = fs.globStatus(new Path(Path.SEPARATOR), falseFilter);
+ assertNull(status);
+
+ status = fs.globStatus(new Path(Path.CUR_DIR), falseFilter);
+ assertNull(status);
+
+ status = fs.globStatus(new Path(USER_DIR), falseFilter);
+ assertNull(status);
+
+ status = fs.globStatus(new Path(USER_DIR, "*"), falseFilter);
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("/x/*"), falseFilter);
+ checkStatus(status);
+
+ status = fs.globStatus(new Path("/x"), falseFilter);
+ assertNull(status);
+
+ status = fs.globStatus(new Path("/x/x"), falseFilter);
+ assertNull(status);
+ }
+
+ private void checkStatus(FileStatus[] status, Path ... expectedMatches) {
+ assertNotNull(status);
+ String[] paths = new String[status.length];
+ for (int i=0; i < status.length; i++) {
+ paths[i] = getPathFromStatus(status[i]);
+ }
+ String got = StringUtils.join(paths, "\n");
+ String expected = StringUtils.join(expectedMatches, "\n");
+ assertEquals(expected, got);
+ }
+
+ private String getPathFromStatus(FileStatus status) {
+ return status.getPath().toUri().getPath();
+ }
+
@Test
public void testPathFilter() throws IOException {
@@ -98,21 +447,7 @@ public class TestGlobPaths {
}
@Test
- public void testGlob() throws Exception {
- //pTestEscape(); // need to wait until HADOOP-1995 is fixed
- pTestJavaRegexSpecialChars();
- pTestCurlyBracket();
- pTestLiteral();
- pTestAny();
- pTestClosure();
- pTestSet();
- pTestRange();
- pTestSetExcl();
- pTestCombination();
- pTestRelativePath();
- }
-
- private void pTestLiteral() throws IOException {
+ public void pTestLiteral() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a2c", USER_DIR+"/abc.d"};
Path[] matchedPath = prepareTesting(USER_DIR+"/abc.d", files);
@@ -123,7 +458,8 @@ public class TestGlobPaths {
}
}
- private void pTestEscape() throws IOException {
+ @Test
+ public void pTestEscape() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/ab\\[c.d"};
Path[] matchedPath = prepareTesting(USER_DIR+"/ab\\[c.d", files);
@@ -134,7 +470,8 @@ public class TestGlobPaths {
}
}
- private void pTestAny() throws IOException {
+ @Test
+ public void pTestAny() throws IOException {
try {
String [] files = new String[] { USER_DIR+"/abc", USER_DIR+"/a2c",
USER_DIR+"/a.c", USER_DIR+"/abcd"};
@@ -148,15 +485,8 @@ public class TestGlobPaths {
}
}
- private void pTestClosure() throws IOException {
- pTestClosure1();
- pTestClosure2();
- pTestClosure3();
- pTestClosure4();
- pTestClosure5();
- }
-
- private void pTestClosure1() throws IOException {
+ @Test
+ public void pTestClosure1() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a", USER_DIR+"/abc",
USER_DIR+"/abc.p", USER_DIR+"/bacd"};
@@ -170,7 +500,8 @@ public class TestGlobPaths {
}
}
- private void pTestClosure2() throws IOException {
+ @Test
+ public void pTestClosure2() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a.", USER_DIR+"/a.txt",
USER_DIR+"/a.old.java", USER_DIR+"/.java"};
@@ -184,7 +515,8 @@ public class TestGlobPaths {
}
}
- private void pTestClosure3() throws IOException {
+ @Test
+ public void pTestClosure3() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a.txt.x", USER_DIR+"/ax",
USER_DIR+"/ab37x", USER_DIR+"/bacd"};
@@ -198,7 +530,8 @@ public class TestGlobPaths {
}
}
- private void pTestClosure4() throws IOException {
+ @Test
+ public void pTestClosure4() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/dir1/file1",
USER_DIR+"/dir2/file2",
@@ -212,7 +545,8 @@ public class TestGlobPaths {
}
}
- private void pTestClosure5() throws IOException {
+ @Test
+ public void pTestClosure5() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/dir1/file1",
USER_DIR+"/file1"};
@@ -224,7 +558,8 @@ public class TestGlobPaths {
}
}
- private void pTestSet() throws IOException {
+ @Test
+ public void pTestSet() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a.c", USER_DIR+"/a.cpp",
USER_DIR+"/a.hlp", USER_DIR+"/a.hxy"};
@@ -238,7 +573,8 @@ public class TestGlobPaths {
}
}
- private void pTestRange() throws IOException {
+ @Test
+ public void pTestRange() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a.d", USER_DIR+"/a.e",
USER_DIR+"/a.f", USER_DIR+"/a.h"};
@@ -252,7 +588,8 @@ public class TestGlobPaths {
}
}
- private void pTestSetExcl() throws IOException {
+ @Test
+ public void pTestSetExcl() throws IOException {
try {
String [] files = new String[] {USER_DIR+"/a.d", USER_DIR+"/a.e",
USER_DIR+"/a.0", USER_DIR+"/a.h"};
@@ -265,7 +602,8 @@ public class TestGlobPaths {
}
}
- private void pTestCombination() throws IOException {
+ @Test
+ public void pTestCombination() throws IOException {
try {
String [] files = new String[] {"/user/aa/a.c", "/user/bb/a.cpp",
"/user1/cc/b.hlp", "/user/dd/a.hxy"};
@@ -277,7 +615,8 @@ public class TestGlobPaths {
}
}
- private void pTestRelativePath() throws IOException {
+ @Test
+ public void pTestRelativePath() throws IOException {
try {
String [] files = new String[] {"a", "abc", "abc.p", "bacd"};
Path[] matchedPath = prepareTesting("a*", files);
@@ -291,7 +630,8 @@ public class TestGlobPaths {
}
/* Test {xx,yy} */
- private void pTestCurlyBracket() throws IOException {
+ @Test
+ public void pTestCurlyBracket() throws IOException {
Path[] matchedPath;
String [] files;
try {
@@ -390,7 +730,8 @@ public class TestGlobPaths {
}
/* test that a path name can contain Java regex special characters */
- private void pTestJavaRegexSpecialChars() throws IOException {
+ @Test
+ public void pTestJavaRegexSpecialChars() throws IOException {
try {
String[] files = new String[] {USER_DIR+"/($.|+)bc", USER_DIR+"/abc"};
Path[] matchedPath = prepareTesting(USER_DIR+"/($.|+)*", files);
@@ -401,6 +742,7 @@ public class TestGlobPaths {
}
}
+
private Path[] prepareTesting(String pattern, String[] files)
throws IOException {
for(int i=0; i<Math.min(NUM_OF_PATHS, files.length); i++) {
@@ -437,8 +779,9 @@ public class TestGlobPaths {
return globResults;
}
- private void cleanupDFS() throws IOException {
- fs.delete(new Path("/user"), true);
+ @After
+ public void cleanupDFS() throws IOException {
+ fs.delete(new Path(USER_DIR), true);
}
}