You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/06/28 09:00:25 UTC
svn commit: r1354832 [5/5] - in
/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project: ./ hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-h...
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java Thu Jun 28 06:59:38 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import static org.junit.Assert.*;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.Collections;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNT
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -118,8 +120,8 @@ public class TestEditLogsDuringFailover
}
}
- @Test
- public void testFailoverFinalizesAndReadsInProgress() throws Exception {
+ private void testFailoverFinalizesAndReadsInProgress(
+ boolean partialTxAtEnd) throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
@@ -130,8 +132,21 @@ public class TestEditLogsDuringFailover
URI sharedUri = cluster.getSharedEditsDir(0, 1);
File sharedDir = new File(sharedUri.getPath(), "current");
FSImageTestUtil.createAbortedLogWithMkdirs(sharedDir, NUM_DIRS_IN_LOG, 1);
+
assertEditFiles(Collections.singletonList(sharedUri),
NNStorage.getInProgressEditsFileName(1));
+ if (partialTxAtEnd) {
+ FileOutputStream outs = null;
+ try {
+ File editLogFile =
+ new File(sharedDir, NNStorage.getInProgressEditsFileName(1));
+ outs = new FileOutputStream(editLogFile, true);
+ outs.write(new byte[] { 0x18, 0x00, 0x00, 0x00 } );
+ LOG.error("editLogFile = " + editLogFile);
+ } finally {
+ IOUtils.cleanup(LOG, outs);
+ }
+ }
// Transition one of the NNs to active
cluster.transitionToActive(0);
@@ -149,7 +164,18 @@ public class TestEditLogsDuringFailover
} finally {
cluster.shutdown();
}
+ }
+
+ @Test
+ public void testFailoverFinalizesAndReadsInProgressSimple()
+ throws Exception {
+ testFailoverFinalizesAndReadsInProgress(false);
+ }
+ @Test
+ public void testFailoverFinalizesAndReadsInProgressWithPartialTxAtEnd()
+ throws Exception {
+ testFailoverFinalizesAndReadsInProgress(true);
}
/**
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java Thu Jun 28 06:59:38 2012
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY;
import static org.junit.Assert.*;
import java.io.File;
@@ -127,6 +129,7 @@ public class TestFailureOfSharedDir {
@Test
public void testFailureOfSharedDir() throws Exception {
Configuration conf = new Configuration();
+ conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, 2000);
// The shared edits dir will automatically be marked required.
MiniDFSCluster cluster = null;
@@ -151,6 +154,15 @@ public class TestFailureOfSharedDir {
assertEquals(0, FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "-w",
true));
+ Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
+ DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2);
+
+ NameNode nn1 = cluster.getNameNode(1);
+ assertTrue(nn1.isStandbyState());
+ assertFalse(
+ "StandBy NameNode should not go to SafeMode on resource unavailability",
+ nn1.isInSafeMode());
+
NameNode nn0 = cluster.getNameNode(0);
nn0.getNamesystem().getFSImage().getEditLog().getJournalSet()
.setRuntimeForTesting(mockRuntime);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java Thu Jun 28 06:59:38 2012
@@ -22,11 +22,14 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.util.Map;
import java.util.HashMap;
import org.junit.Test;
import org.junit.Before;
+
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.commons.logging.Log;
@@ -34,12 +37,12 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer;
+import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper;
public class TestOfflineEditsViewer {
-
private static final Log LOG = LogFactory.getLog(TestOfflineEditsViewer.class);
private static final Map<FSEditLogOpCodes, Boolean> obsoleteOpCodes =
@@ -97,8 +100,8 @@ public class TestOfflineEditsViewer {
String editsReparsed = cacheDir + "/editsReparsed";
// parse to XML then back to binary
- runOev(edits, editsParsedXml, "xml");
- runOev(editsParsedXml, editsReparsed, "binary");
+ assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
+ assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
// judgment time
assertTrue(
@@ -115,6 +118,42 @@ public class TestOfflineEditsViewer {
}
@Test
+ public void testRecoveryMode() throws IOException {
+ LOG.info("START - testing with generated edits");
+
+ nnHelper.startCluster(buildDir + "/dfs/");
+
+ // edits generated by nnHelper (MiniDFSCluster), should have all op codes
+ // binary, XML, reparsed binary
+ String edits = nnHelper.generateEdits();
+
+ // Corrupt the file by truncating the end
+ FileChannel editsFile = new FileOutputStream(edits, true).getChannel();
+ editsFile.truncate(editsFile.size() - 5);
+
+ String editsParsedXml = cacheDir + "/editsRecoveredParsed.xml";
+ String editsReparsed = cacheDir + "/editsRecoveredReparsed";
+ String editsParsedXml2 = cacheDir + "/editsRecoveredParsed2.xml";
+
+ // Can't read the corrupted file without recovery mode
+ assertEquals(-1, runOev(edits, editsParsedXml, "xml", false));
+
+ // parse to XML then back to binary
+ assertEquals(0, runOev(edits, editsParsedXml, "xml", true));
+ assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false));
+ assertEquals(0, runOev(editsReparsed, editsParsedXml2, "xml", false));
+
+ // judgment time
+ assertTrue("Test round trip",
+ filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2));
+
+ // removes edits so do this at the end
+ nnHelper.shutdownCluster();
+
+ LOG.info("END");
+ }
+
+ @Test
public void testStored() throws IOException {
LOG.info("START - testing with stored reference edits");
@@ -128,8 +167,9 @@ public class TestOfflineEditsViewer {
String editsStoredXml = cacheDir + "/editsStored.xml";
// parse to XML then back to binary
- runOev(editsStored, editsStoredParsedXml, "xml");
- runOev(editsStoredParsedXml, editsStoredReparsed, "binary");
+ assertEquals(0, runOev(editsStored, editsStoredParsedXml, "xml", false));
+ assertEquals(0, runOev(editsStoredParsedXml, editsStoredReparsed,
+ "binary", false));
// judgement time
assertTrue(
@@ -151,14 +191,18 @@ public class TestOfflineEditsViewer {
* @param inFilename input edits filename
* @param outFilename oputput edits filename
*/
- private void runOev(String inFilename, String outFilename, String processor)
- throws IOException {
+ private int runOev(String inFilename, String outFilename, String processor,
+ boolean recovery) throws IOException {
LOG.info("Running oev [" + inFilename + "] [" + outFilename + "]");
OfflineEditsViewer oev = new OfflineEditsViewer();
- if (oev.go(inFilename, outFilename, processor, true, false, null) != 0)
- throw new RuntimeException("oev failed");
+ Flags flags = new Flags();
+ flags.setPrintToScreen();
+ if (recovery) {
+ flags.setRecoveryMode();
+ }
+ return oev.go(inFilename, outFilename, processor, flags, null);
}
/**
@@ -172,7 +216,7 @@ public class TestOfflineEditsViewer {
FileOutputStream fout = new FileOutputStream(outFilename);
StatisticsEditsVisitor visitor = new StatisticsEditsVisitor(fout);
OfflineEditsViewer oev = new OfflineEditsViewer();
- if (oev.go(inFilename, outFilename, "stats", false, false, visitor) != 0)
+ if (oev.go(inFilename, outFilename, "stats", new Flags(), visitor) != 0)
return false;
LOG.info("Statistics for " + inFilename + "\n" +
visitor.getStatisticsString());
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Thu Jun 28 06:59:38 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -286,6 +287,10 @@ public class TestWebHdfsFileSystemContra
final Path root = new Path("/");
final Path dir = new Path("/test/testUrl");
assertTrue(webhdfs.mkdirs(dir));
+ final Path file = new Path("/test/file");
+ final FSDataOutputStream out = webhdfs.create(file);
+ out.write(1);
+ out.close();
{//test GETHOMEDIRECTORY
final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root);
@@ -351,5 +356,47 @@ public class TestWebHdfsFileSystemContra
{//test append.
AppendTestUtil.testAppend(fs, new Path(dir, "append"));
}
+
+ {//test NamenodeRpcAddressParam not set.
+ final HttpOpParam.Op op = PutOpParam.Op.CREATE;
+ final URL url = webhdfs.toUrl(op, dir);
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod(op.getType().toString());
+ conn.setDoOutput(false);
+ conn.setInstanceFollowRedirects(false);
+ conn.connect();
+ final String redirect = conn.getHeaderField("Location");
+ conn.disconnect();
+
+ //remove NamenodeRpcAddressParam
+ WebHdfsFileSystem.LOG.info("redirect = " + redirect);
+ final int i = redirect.indexOf(NamenodeRpcAddressParam.NAME);
+ final int j = redirect.indexOf("&", i);
+ String modified = redirect.substring(0, i - 1) + redirect.substring(j);
+ WebHdfsFileSystem.LOG.info("modified = " + modified);
+
+ //connect to datanode
+ conn = (HttpURLConnection)new URL(modified).openConnection();
+ conn.setRequestMethod(op.getType().toString());
+ conn.setDoOutput(op.getDoOutput());
+ conn.connect();
+ assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode());
+ }
+
+ {//test jsonParse with non-json type.
+ final HttpOpParam.Op op = GetOpParam.Op.OPEN;
+ final URL url = webhdfs.toUrl(op, file);
+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod(op.getType().toString());
+ conn.connect();
+
+ try {
+ WebHdfsFileSystem.jsonParse(conn, false);
+ fail();
+ } catch(IOException ioe) {
+ WebHdfsFileSystem.LOG.info("GOOD", ioe);
+ }
+ conn.disconnect();
+ }
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java Thu Jun 28 06:59:38 2012
@@ -40,6 +40,12 @@ import org.junit.Assert;
public class WebHdfsTestUtil {
public static final Log LOG = LogFactory.getLog(WebHdfsTestUtil.class);
+ public static Configuration createConf() {
+ final Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+ return conf;
+ }
+
public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf
) throws IOException, URISyntaxException {
final String uri = WebHdfsFileSystem.SCHEME + "://"
@@ -49,7 +55,7 @@ public class WebHdfsTestUtil {
public static WebHdfsFileSystem getWebHdfsFileSystemAs(
final UserGroupInformation ugi, final Configuration conf
- ) throws IOException, URISyntaxException, InterruptedException {
+ ) throws IOException, InterruptedException {
return ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
@Override
public WebHdfsFileSystem run() throws Exception {
@@ -70,7 +76,7 @@ public class WebHdfsTestUtil {
final int expectedResponseCode) throws IOException {
conn.connect();
Assert.assertEquals(expectedResponseCode, conn.getResponseCode());
- return WebHdfsFileSystem.jsonParse(conn.getInputStream());
+ return WebHdfsFileSystem.jsonParse(conn, false);
}
public static HttpURLConnection twoStepWrite(HttpURLConnection conn,
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Thu Jun 28 06:59:38 2012
@@ -15470,7 +15470,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>For input string: "a5"</expected-output>
+ <expected-output>setSpaceQuota: "a5" is not a valid value for a quota.</expected-output>
</comparator>
</comparators>
</test>
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml Thu Jun 28 06:59:38 2012
@@ -34,7 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
<module>hadoop-hdfs</module>
<module>hadoop-hdfs-httpfs</module>
<module>hadoop-hdfs/src/contrib/bkjournal</module>
- <module>hadoop-hdfs/src/contrib/fuse-dfs</module>
+ <module>hadoop-hdfs-raid</module>
</modules>
<build>