You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/08/13 23:31:47 UTC
svn commit: r1372630 - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/datanode/
src/main/java/...
Author: todd
Date: Mon Aug 13 21:31:44 2012
New Revision: 1372630
URL: http://svn.apache.org/viewvc?rev=1372630&view=rev
Log:
Merge trunk into branch
Modified:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/TestFuseDFS.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1371514-1372629
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Aug 13 21:31:44 2012
@@ -104,9 +104,6 @@ Trunk (unreleased changes)
HDFS-3573. Supply NamespaceInfo when instantiating JournalManagers (todd)
- HDFS-3190. Simple refactors in existing NN code to assist
- QuorumJournalManager extension. (todd)
-
HDFS-3630 Modify TestPersistBlocks to use both flush and hflush (sanjay)
HDFS-3768. Exception in TestJettyHelper is incorrect.
@@ -114,6 +111,9 @@ Trunk (unreleased changes)
HDFS-3695. Genericize format() to non-file JournalManagers. (todd)
+ HDFS-3789. JournalManager#format() should be able to throw IOException
+ (Ivan Kelly via todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -127,10 +127,6 @@ Trunk (unreleased changes)
HDFS-2314. MRV1 test compilation broken after HDFS-2197 (todd)
- HDFS-2330. In NNStorage and FSImagePreTransactionalStorageInspector,
- IOExceptions of stream closures can mask root exceptions. (Uma Maheswara
- Rao G via szetszwo)
-
HDFS-46. Change default namespace quota of root directory from
Integer.MAX_VALUE to Long.MAX_VALUE. (Uma Maheswara Rao G via szetszwo)
@@ -380,6 +376,11 @@ Branch-2 ( Unreleased changes )
HDFS-3634. Add self-contained, mavenized fuse_dfs test. (Colin Patrick
McCabe via atm)
+ HDFS-3190. Simple refactors in existing NN code to assist
+ QuorumJournalManager extension. (todd)
+
+ HDFS-3276. initializeSharedEdits should have a -nonInteractive flag (todd)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
@@ -570,9 +571,6 @@ Branch-2 ( Unreleased changes )
HDFS-3756. DelegationTokenFetcher creates 2 HTTP connections, the second
one not properly configured. (tucu)
- HDFS-3719. Re-enable append-related tests in TestFileConcurrentReader.
- (Andrew Wang via atm)
-
HDFS-3579. libhdfs: fix exception handling. (Colin Patrick McCabe via atm)
HDFS-3754. BlockSender doesn't shutdown ReadaheadPool threads. (eli)
@@ -583,6 +581,12 @@ Branch-2 ( Unreleased changes )
HDFS-3721. hsync support broke wire compatibility. (todd and atm)
+ HDFS-3758. TestFuseDFS test failing. (Colin Patrick McCabe via eli)
+
+ HDFS-2330. In NNStorage and FSImagePreTransactionalStorageInspector,
+ IOExceptions of stream closures can mask root exceptions. (Uma Maheswara
+ Rao G via szetszwo)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Mon Aug 13 21:31:44 2012
@@ -272,7 +272,7 @@ public class BookKeeperJournalManager im
}
@Override
- public void format(NamespaceInfo ns) {
+ public void format(NamespaceInfo ns) throws IOException {
// Currently, BKJM automatically formats itself when first accessed.
// TODO: change over to explicit formatting so that the admin can
// clear out the BK storage when reformatting a cluster.
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1371514-1372629
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java Mon Aug 13 21:31:44 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.da
import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URL;
import java.net.URLEncoder;
@@ -37,7 +36,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -45,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -140,7 +139,7 @@ public class DatanodeJspHelper {
DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock, conf);
String fqdn = canonicalize(chosenNode.getIpAddr());
int datanodePort = chosenNode.getXferPort();
- String redirectLocation = "http://" + fqdn + ":"
+ String redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":"
+ chosenNode.getInfoPort() + "/browseBlock.jsp?blockId="
+ firstBlock.getBlock().getBlockId() + "&blockSize="
+ firstBlock.getBlock().getNumBytes() + "&genstamp="
@@ -220,7 +219,7 @@ public class DatanodeJspHelper {
JspHelper.addTableFooter(out);
}
}
- out.print("<br><a href=\"http://"
+ out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ canonicalize(nnAddr) + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -296,7 +295,7 @@ public class DatanodeJspHelper {
Long.MAX_VALUE).getLocatedBlocks();
// Add the various links for looking at the file contents
// URL for downloading the full file
- String downloadUrl = "http://" + req.getServerName() + ":"
+ String downloadUrl = HttpConfig.getSchemePrefix() + req.getServerName() + ":"
+ req.getServerPort() + "/streamFile" + ServletUtil.encodePath(filename)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr, true)
+ JspHelper.getDelegationTokenUrlParam(tokenString);
@@ -314,7 +313,7 @@ public class DatanodeJspHelper {
return;
}
String fqdn = canonicalize(chosenNode.getIpAddr());
- String tailUrl = "http://" + fqdn + ":" + chosenNode.getInfoPort()
+ String tailUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + chosenNode.getInfoPort()
+ "/tail.jsp?filename=" + URLEncoder.encode(filename, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
+ "&chunkSizeToView=" + chunkSizeToView
@@ -363,7 +362,7 @@ public class DatanodeJspHelper {
String datanodeAddr = locs[j].getXferAddr();
datanodePort = locs[j].getXferPort();
fqdn = canonicalize(locs[j].getIpAddr());
- String blockUrl = "http://" + fqdn + ":" + locs[j].getInfoPort()
+ String blockUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + locs[j].getInfoPort()
+ "/browseBlock.jsp?blockId=" + blockidstring
+ "&blockSize=" + blockSize
+ "&filename=" + URLEncoder.encode(filename, "UTF-8")
@@ -374,7 +373,7 @@ public class DatanodeJspHelper {
+ JspHelper.getDelegationTokenUrlParam(tokenString)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
- String blockInfoUrl = "http://" + nnCanonicalName + ":"
+ String blockInfoUrl = HttpConfig.getSchemePrefix() + nnCanonicalName + ":"
+ namenodeInfoPort
+ "/block_info_xml.jsp?blockId=" + blockidstring;
out.print("<td> </td><td><a href=\"" + blockUrl + "\">"
@@ -385,7 +384,7 @@ public class DatanodeJspHelper {
}
out.println("</table>");
out.print("<hr>");
- out.print("<br><a href=\"http://"
+ out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ nnCanonicalName + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -485,7 +484,7 @@ public class DatanodeJspHelper {
String parent = new File(filename).getParent();
JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, parent, nnAddr);
out.print("<hr>");
- out.print("<a href=\"http://"
+ out.print("<a href=\"" + HttpConfig.getSchemePrefix()
+ req.getServerName() + ":" + req.getServerPort()
+ "/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
@@ -533,7 +532,7 @@ public class DatanodeJspHelper {
}
String nextUrl = null;
if (nextBlockIdStr != null) {
- nextUrl = "http://" + canonicalize(nextHost) + ":" + nextPort
+ nextUrl = HttpConfig.getSchemePrefix() + canonicalize(nextHost) + ":" + nextPort
+ "/browseBlock.jsp?blockId=" + nextBlockIdStr
+ "&blockSize=" + nextBlockSize
+ "&startOffset=" + nextStartOffset
@@ -588,7 +587,7 @@ public class DatanodeJspHelper {
String prevUrl = null;
if (prevBlockIdStr != null) {
- prevUrl = "http://" + canonicalize(prevHost) + ":" + prevPort
+ prevUrl = HttpConfig.getSchemePrefix() + canonicalize(prevHost) + ":" + prevPort
+ "/browseBlock.jsp?blockId=" + prevBlockIdStr
+ "&blockSize=" + prevBlockSize
+ "&startOffset=" + prevStartOffset
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java Mon Aug 13 21:31:44 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
@@ -823,7 +824,7 @@ class ClusterJspHelper {
doc.startTag("item");
doc.attribute("label", label);
doc.attribute("value", value);
- doc.attribute("link", "http://" + url);
+ doc.attribute("link", HttpConfig.getSchemePrefix() + url);
doc.endTag(); // item
}
@@ -883,7 +884,7 @@ class ClusterJspHelper {
private static String queryMbean(String httpAddress, Configuration conf)
throws IOException {
- URL url = new URL("http://"+httpAddress+JMX_QRY);
+ URL url = new URL(HttpConfig.getSchemePrefix() + httpAddress+JMX_QRY);
return readOutput(url);
}
/**
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Mon Aug 13 21:31:44 2012
@@ -82,7 +82,7 @@ public class FileJournalManager implemen
public void close() throws IOException {}
@Override
- public void format(NamespaceInfo ns) {
+ public void format(NamespaceInfo ns) throws IOException {
// Formatting file journals is done by the StorageDirectory
// format code, since they may share their directory with
// checkpoints, etc.
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java Mon Aug 13 21:31:44 2012
@@ -41,7 +41,7 @@ public interface JournalManager extends
* Format the underlying storage, removing any previously
* stored data.
*/
- void format(NamespaceInfo ns);
+ void format(NamespaceInfo ns) throws IOException;
/**
* Begin writing to a new segment of the log stream, which starts at
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java Mon Aug 13 21:31:44 2012
@@ -174,7 +174,7 @@ public class JournalSet implements Journ
}
@Override
- public void format(NamespaceInfo nsInfo) {
+ public void format(NamespaceInfo nsInfo) throws IOException {
// The iteration is done by FSEditLog itself
throw new UnsupportedOperationException();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Aug 13 21:31:44 2012
@@ -894,7 +894,10 @@ public class NameNode {
StartupOption.ROLLBACK.getName() + "] | [" +
StartupOption.FINALIZE.getName() + "] | [" +
StartupOption.IMPORT.getName() + "] | [" +
- StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" +
+ StartupOption.INITIALIZESHAREDEDITS.getName() +
+ " [" + StartupOption.FORCE.getName() + "] [" +
+ StartupOption.NONINTERACTIVE.getName() + "]" +
+ "] | [" +
StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" +
StartupOption.RECOVER.getName() + " [ " +
StartupOption.FORCE.getName() + " ] ]");
@@ -964,6 +967,16 @@ public class NameNode {
return startOpt;
} else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.INITIALIZESHAREDEDITS;
+ for (i = i + 1 ; i < argsLen; i++) {
+ if (StartupOption.NONINTERACTIVE.getName().equals(args[i])) {
+ startOpt.setInteractiveFormat(false);
+ } else if (StartupOption.FORCE.getName().equals(args[i])) {
+ startOpt.setForceFormat(true);
+ } else {
+ LOG.fatal("Invalid argument: " + args[i]);
+ return null;
+ }
+ }
return startOpt;
} else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) {
if (startOpt != StartupOption.REGULAR) {
@@ -1073,7 +1086,9 @@ public class NameNode {
return null; // avoid warning
}
case INITIALIZESHAREDEDITS: {
- boolean aborted = initializeSharedEdits(conf, false, true);
+ boolean aborted = initializeSharedEdits(conf,
+ startOpt.getForceFormat(),
+ startOpt.getInteractiveFormat());
terminate(aborted ? 1 : 0);
return null; // avoid warning
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java Mon Aug 13 21:31:44 2012
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
@@ -413,7 +414,7 @@ class NamenodeJspHelper {
}
String addr = NetUtils.getHostPortString(nn.getNameNodeAddress());
String fqdn = InetAddress.getByName(nodeToRedirect).getCanonicalHostName();
- redirectLocation = "http://" + fqdn + ":" + redirectPort
+ redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":" + redirectPort
+ "/browseDirectory.jsp?namenodeInfoPort="
+ nn.getHttpAddress().getPort() + "&dir=/"
+ (tokenString == null ? "" :
@@ -462,7 +463,8 @@ class NamenodeJspHelper {
String suffix, boolean alive, int nnHttpPort, String nnaddr)
throws IOException {
// from nn_browsedfscontent.jsp:
- String url = "http://" + d.getHostName() + ":" + d.getInfoPort()
+ String url = HttpConfig.getSchemePrefix() + d.getHostName() + ":"
+ + d.getInfoPort()
+ "/browseDirectory.jsp?namenodeInfoPort=" + nnHttpPort + "&dir="
+ URLEncoder.encode("/", "UTF-8")
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Mon Aug 13 21:31:44 2012
@@ -32,12 +32,12 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
@@ -206,7 +206,8 @@ public class TransferFsImage {
String queryString, List<File> localPaths,
Storage dstStorage, boolean getChecksum) throws IOException {
- String str = "http://" + nnHostPort + "/getimage?" + queryString;
+ String str = HttpConfig.getSchemePrefix() + nnHostPort + "/getimage?" +
+ queryString;
LOG.info("Opening connection to " + str);
//
// open connection to remote server
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java Mon Aug 13 21:31:44 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.Distribute
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -226,7 +227,7 @@ public class DFSck extends Configured im
}
private int doWork(final String[] args) throws IOException {
- final StringBuilder url = new StringBuilder("http://");
+ final StringBuilder url = new StringBuilder(HttpConfig.getSchemePrefix());
String namenodeAddress = getCurrentNamenodeAddress();
if (namenodeAddress == null) {
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1371514-1372629
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/TestFuseDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/TestFuseDFS.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/TestFuseDFS.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/TestFuseDFS.java Mon Aug 13 21:31:44 2012
@@ -44,6 +44,7 @@ public class TestFuseDFS {
private static MiniDFSCluster cluster;
private static FileSystem fs;
+ private static Process fuseProcess;
private static Runtime r;
private static String mountPoint;
@@ -137,8 +138,28 @@ public class TestFuseDFS {
assertEquals("File content differs", expectedContents, s);
}
+ private static class RedirectToStdoutThread extends Thread {
+ private InputStream is;
+
+ RedirectToStdoutThread(InputStream is) {
+ this.is = is;
+ }
+ public void run() {
+ try {
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+ String line=null;
+ while ( (line = br.readLine()) != null) {
+ LOG.error("FUSE_LINE:" + line);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
/** Run a fuse-dfs process to mount the given DFS */
- private static void establishMount(URI uri) throws IOException {
+ private static Process establishMount(URI uri) throws IOException {
Runtime r = Runtime.getRuntime();
String cp = System.getProperty("java.class.path");
@@ -163,6 +184,8 @@ public class TestFuseDFS {
"-obig_writes", // Allow >4kb writes
"-oentry_timeout=0.1", // Don't cache dents long
"-oattribute_timeout=0.1", // Don't cache attributes long
+ "-ononempty", // Don't complain about junk in mount point
+ "-f", // Don't background the process
"-ordbuffer=32768", // Read buffer size in kb
"rw"
};
@@ -178,17 +201,35 @@ public class TestFuseDFS {
execAssertSucceeds("mkdir -p " + mountPoint);
// Mount the mini cluster
+ String cmdStr = "";
+ for (String c : mountCmd) {
+ cmdStr += (" " + c);
+ }
+ LOG.info("now mounting with:" + cmdStr);
+ Process fuseProcess = r.exec(mountCmd, env);
+ RedirectToStdoutThread stdoutThread =
+ new RedirectToStdoutThread(fuseProcess.getInputStream());
+ RedirectToStdoutThread stderrThread =
+ new RedirectToStdoutThread(fuseProcess.getErrorStream());
+ stdoutThread.start();
+ stderrThread.start();
+ // Wait for fusermount to start up, so that we know we're operating on the
+ // FUSE FS when we run the tests.
try {
- Process fuseProcess = r.exec(mountCmd, env);
- assertEquals(0, fuseProcess.waitFor());
- } catch (InterruptedException ie) {
- fail("Failed to mount");
+ Thread.sleep(50000);
+ } catch (InterruptedException e) {
}
+ return fuseProcess;
}
/** Tear down the fuse-dfs process and mount */
private static void teardownMount() throws IOException {
execWaitRet("fusermount -u " + mountPoint);
+ try {
+ assertEquals(0, fuseProcess.waitFor()); // fuse_dfs should exit cleanly
+ } catch (InterruptedException e) {
+ fail("interrupted while waiting for fuse_dfs process to exit.");
+ }
}
@BeforeClass
@@ -200,7 +241,7 @@ public class TestFuseDFS {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitClusterUp();
fs = cluster.getFileSystem();
- establishMount(fs.getUri());
+ fuseProcess = establishMount(fs.getUri());
}
@AfterClass
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1371514-1372629
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1371514-1372629
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1371514-1372629
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1371514-1372629
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java Mon Aug 13 21:31:44 2012
@@ -288,8 +288,10 @@ public class TestFileConcurrentReader {
runTestUnfinishedBlockCRCError(true, SyncType.SYNC, SMALL_WRITE_SIZE);
}
+ // fails due to issue w/append, disable
+ @Ignore
@Test
- public void testUnfinishedBlockCRCErrorTransferToAppend()
+ public void _testUnfinishedBlockCRCErrorTransferToAppend()
throws IOException {
runTestUnfinishedBlockCRCError(true, SyncType.APPEND, DEFAULT_WRITE_SIZE);
}
@@ -305,8 +307,10 @@ public class TestFileConcurrentReader {
runTestUnfinishedBlockCRCError(false, SyncType.SYNC, SMALL_WRITE_SIZE);
}
+ // fails due to issue w/append, disable
+ @Ignore
@Test
- public void testUnfinishedBlockCRCErrorNormalTransferAppend()
+ public void _testUnfinishedBlockCRCErrorNormalTransferAppend()
throws IOException {
runTestUnfinishedBlockCRCError(false, SyncType.APPEND, DEFAULT_WRITE_SIZE);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1372630&r1=1372629&r2=1372630&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Mon Aug 13 21:31:44 2012
@@ -154,7 +154,7 @@ public class TestGenericJournalConf {
}
@Override
- public void format(NamespaceInfo nsInfo) {
+ public void format(NamespaceInfo nsInfo) throws IOException {
formatCalled = true;
}