You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cn...@apache.org on 2013/06/18 07:10:57 UTC
svn commit: r1494025 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: cnauroth
Date: Tue Jun 18 05:10:57 2013
New Revision: 1494025
URL: http://svn.apache.org/r1494025
Log:
HDFS-4818. Merging change r1494024 from branch-2 to branch-2.1-beta.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jun 18 05:10:57 2013
@@ -499,6 +499,9 @@ Release 2.1.0-beta - UNRELEASED
HDFS-4783. TestDelegationTokensWithHA#testHAUtilClonesDelegationTokens fails
on Windows. (cnauroth)
+ HDFS-4818. Several HDFS tests that attempt to make directories unusable do
+ not work correctly on Windows. (cnauroth)
+
BREAKDOWN OF HDFS-2802 HDFS SNAPSHOT SUBTASKS AND RELATED JIRAS
HDFS-4076. Support snapshot of single files. (szetszwo)
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Jun 18 05:10:57 2013
@@ -842,8 +842,8 @@ public class NNStorage extends Storage i
String absPath = f.getAbsolutePath();
for (StorageDirectory sd : storageDirs) {
String dirPath = sd.getRoot().getAbsolutePath();
- if (!dirPath.endsWith("/")) {
- dirPath += "/";
+ if (!dirPath.endsWith(File.separator)) {
+ dirPath += File.separator;
}
if (absPath.startsWith(dirPath)) {
reportErrorsOnDirectory(sd);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Jun 18 05:10:57 2013
@@ -873,9 +873,13 @@ public class TestCheckpoint {
savedSd.lock();
fail("Namenode should not be able to lock a storage that is already locked");
} catch (IOException ioe) {
- String jvmName = ManagementFactory.getRuntimeMXBean().getName();
- assertTrue("Error message does not include JVM name '" + jvmName
- + "'", logs.getOutput().contains(jvmName));
+ // cannot read lock file on Windows, so message cannot get JVM name
+ String lockingJvmName = Path.WINDOWS ? "" :
+ " " + ManagementFactory.getRuntimeMXBean().getName();
+ String expectedLogMessage = "It appears that another namenode"
+ + lockingJvmName + " has already locked the storage directory";
+ assertTrue("Log output does not contain expected log message: "
+ + expectedLogMessage, logs.getOutput().contains(expectedLogMessage));
}
} finally {
cleanup(cluster);
@@ -2038,7 +2042,7 @@ public class TestCheckpoint {
StorageDirectory sd0 = storage.getStorageDir(0);
assertEquals(NameNodeDirType.IMAGE, sd0.getStorageDirType());
currentDir = sd0.getCurrentDir();
- FileUtil.setExecutable(currentDir, false);
+ assertEquals(0, FileUtil.chmod(currentDir.getAbsolutePath(), "000"));
// Try to upload checkpoint -- this should fail since there are no
// valid storage dirs
@@ -2051,7 +2055,7 @@ public class TestCheckpoint {
}
// Restore the good dir
- FileUtil.setExecutable(currentDir, true);
+ assertEquals(0, FileUtil.chmod(currentDir.getAbsolutePath(), "755"));
nn.restoreFailedStorage("true");
nn.rollEditLog();
@@ -2062,7 +2066,7 @@ public class TestCheckpoint {
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
} finally {
if (currentDir != null) {
- FileUtil.setExecutable(currentDir, true);
+ FileUtil.chmod(currentDir.getAbsolutePath(), "755");
}
cleanup(secondary);
secondary = null;
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Tue Jun 18 05:10:57 2013
@@ -242,8 +242,8 @@ public class TestFileJournalManager {
try {
jm.finalizeLogSegment(0, 1);
} finally {
- assertTrue(storage.getRemovedStorageDirs().contains(sd));
FileUtil.chmod(sdRootPath, "+w", true);
+ assertTrue(storage.getRemovedStorageDirs().contains(sd));
}
}
@@ -440,8 +440,12 @@ public class TestFileJournalManager {
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
EditLogInputStream elis = getJournalInputStream(jm, 5, true);
- FSEditLogOp op = elis.readOp();
- assertEquals("read unexpected op", op.getTransactionId(), 5);
+ try {
+ FSEditLogOp op = elis.readOp();
+ assertEquals("read unexpected op", op.getTransactionId(), 5);
+ } finally {
+ IOUtils.cleanup(LOG, elis);
+ }
}
/**
@@ -464,9 +468,13 @@ public class TestFileJournalManager {
assertEquals(100, getNumberOfTransactions(jm, 1, false, false));
EditLogInputStream elis = getJournalInputStream(jm, 90, false);
- FSEditLogOp lastReadOp = null;
- while ((lastReadOp = elis.readOp()) != null) {
- assertTrue(lastReadOp.getTransactionId() <= 100);
+ try {
+ FSEditLogOp lastReadOp = null;
+ while ((lastReadOp = elis.readOp()) != null) {
+ assertTrue(lastReadOp.getTransactionId() <= 100);
+ }
+ } finally {
+ IOUtils.cleanup(LOG, elis);
}
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Tue Jun 18 05:10:57 2013
@@ -106,6 +106,9 @@ public class TestFsck {
static final Pattern numCorruptBlocksPattern = Pattern.compile(
".*Corrupt blocks:\t\t([0123456789]*).*");
+ private static final String LINE_SEPARATOR =
+ System.getProperty("line.separator");
+
static String runFsck(Configuration conf, int expectedErrCode,
boolean checkErrorCode,String... path)
throws Exception {
@@ -321,7 +324,7 @@ public class TestFsck {
while (true) {
outStr = runFsck(conf, 1, false, "/");
String numCorrupt = null;
- for (String line : outStr.split("\n")) {
+ for (String line : outStr.split(LINE_SEPARATOR)) {
Matcher m = numCorruptBlocksPattern.matcher(line);
if (m.matches()) {
numCorrupt = m.group(1);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionFunctional.java Tue Jun 18 05:10:57 2013
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.ser
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
import static org.apache.hadoop.test.GenericTestUtils.assertGlobEquals;
+import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
@@ -59,7 +60,7 @@ public class TestNNStorageRetentionFunct
*/
@Test
public void testPurgingWithNameEditsDirAfterFailure()
- throws IOException {
+ throws Exception {
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);
@@ -107,10 +108,10 @@ public class TestNNStorageRetentionFunct
getInProgressEditsFileName(5));
LOG.info("Failing first storage dir by chmodding it");
- FileUtil.setExecutable(sd0, false);
+ assertEquals(0, FileUtil.chmod(cd0.getAbsolutePath(), "000"));
doSaveNamespace(nn);
LOG.info("Restoring accessibility of first storage dir");
- FileUtil.setExecutable(sd0, true);
+ assertEquals(0, FileUtil.chmod(cd0.getAbsolutePath(), "755"));
LOG.info("nothing should have been purged in first storage dir");
assertGlobEquals(cd0, "fsimage_\\d*",
@@ -139,7 +140,7 @@ public class TestNNStorageRetentionFunct
assertGlobEquals(cd0, "edits_.*",
getInProgressEditsFileName(9));
} finally {
- FileUtil.setExecutable(sd0, true);
+ FileUtil.chmod(cd0.getAbsolutePath(), "755");
LOG.info("Shutting down...");
if (cluster != null) {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1494025&r1=1494024&r2=1494025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Tue Jun 18 05:10:57 2013
@@ -130,7 +130,8 @@ public class TestNameNodeMXBean {
// This will cause the first dir to fail.
File failedNameDir = new File(nameDirUris.toArray(new URI[0])[0]);
- assertEquals(0, FileUtil.chmod(failedNameDir.getAbsolutePath(), "000"));
+ assertEquals(0, FileUtil.chmod(
+ new File(failedNameDir, "current").getAbsolutePath(), "000"));
cluster.getNameNodeRpc().rollEditLog();
nameDirStatuses = (String) (mbs.getAttribute(mxbeanName,
@@ -150,7 +151,8 @@ public class TestNameNodeMXBean {
} finally {
if (cluster != null) {
for (URI dir : cluster.getNameDirs(0)) {
- FileUtil.chmod(new File(dir).toString(), "700");
+ FileUtil.chmod(
+ new File(new File(dir), "current").getAbsolutePath(), "755");
}
cluster.shutdown();
}