You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/01/27 20:31:53 UTC
svn commit: r1561802 [3/3] - in
/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bk...
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Mon Jan 27 19:31:47 2014
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.junit.Assume;
import org.junit.Before;
@@ -764,4 +765,37 @@ public class TestDFSUtil {
assertEquals(4*24*60*60*1000l, DFSUtil.parseRelativeTime("4d"));
assertEquals(999*24*60*60*1000l, DFSUtil.parseRelativeTime("999d"));
}
+
+ @Test
+ public void testAssertAllResultsEqual() {
+ checkAllResults(new Long[]{}, true);
+ checkAllResults(new Long[]{1l}, true);
+ checkAllResults(new Long[]{1l, 1l}, true);
+ checkAllResults(new Long[]{1l, 1l, 1l}, true);
+ checkAllResults(new Long[]{new Long(1), new Long(1)}, true);
+ checkAllResults(new Long[]{null, null, null}, true);
+
+ checkAllResults(new Long[]{1l, 2l}, false);
+ checkAllResults(new Long[]{2l, 1l}, false);
+ checkAllResults(new Long[]{1l, 2l, 1l}, false);
+ checkAllResults(new Long[]{2l, 1l, 1l}, false);
+ checkAllResults(new Long[]{1l, 1l, 2l}, false);
+ checkAllResults(new Long[]{1l, null}, false);
+ checkAllResults(new Long[]{null, 1l}, false);
+ checkAllResults(new Long[]{1l, null, 1l}, false);
+ }
+
+ private static void checkAllResults(Long[] toCheck, boolean shouldSucceed) {
+ if (shouldSucceed) {
+ DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck));
+ } else {
+ try {
+ DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck));
+ fail("Should not have succeeded with input: " +
+ Arrays.toString(toCheck));
+ } catch (AssertionError ae) {
+ GenericTestUtils.assertExceptionContains("Not all elements match", ae);
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java Mon Jan 27 19:31:47 2014
@@ -19,20 +19,28 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestInterDatanodeProtocol;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Test;
public class TestLeaseRecovery {
@@ -148,4 +156,55 @@ public class TestLeaseRecovery {
if (cluster != null) {cluster.shutdown();}
}
}
+
+ /**
+ * Block Recovery when the meta file not having crcs for all chunks in block
+ * file
+ */
+ @Test
+ public void testBlockRecoveryWithLessMetafile() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .build();
+ Path file = new Path("/testRecoveryFile");
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ FSDataOutputStream out = dfs.create(file);
+ int count = 0;
+ while (count < 2 * 1024 * 1024) {
+ out.writeBytes("Data");
+ count += 4;
+ }
+ out.hsync();
+ // abort the original stream
+ ((DFSOutputStream) out.getWrappedStream()).abort();
+
+ LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(
+ file.toString(), 0, count);
+ ExtendedBlock block = locations.get(0).getBlock();
+ DataNode dn = cluster.getDataNodes().get(0);
+ BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null);
+ File metafile = new File(localPathInfo.getMetaPath());
+ assertTrue(metafile.exists());
+
+ // reduce the block meta file size
+ RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
+ raf.setLength(metafile.length() - 20);
+ raf.close();
+
+ // restart DN to make replica to RWR
+ DataNodeProperties dnProp = cluster.stopDataNode(0);
+ cluster.restartDataNode(dnProp, true);
+
+ // try to recover the lease
+ DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem
+ .newInstance(cluster.getConfiguration(0));
+ count = 0;
+ while (++count < 10 && !newdfs.recoverLease(file)) {
+ Thread.sleep(1000);
+ }
+ assertTrue("File should be closed", newdfs.recoverLease(file));
+
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java Mon Jan 27 19:31:47 2014
@@ -167,8 +167,16 @@ public class MiniJournalCluster {
return new File(baseDir, "journalnode-" + idx).getAbsoluteFile();
}
+ public File getJournalDir(int idx, String jid) {
+ return new File(getStorageDir(idx), jid);
+ }
+
public File getCurrentDir(int idx, String jid) {
- return new File(new File(getStorageDir(idx), jid), "current");
+ return new File(getJournalDir(idx, jid), "current");
+ }
+
+ public File getPreviousDir(int idx, String jid) {
+ return new File(getJournalDir(idx, jid), "previous");
}
public JournalNode getJournalNode(int i) {
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java Mon Jan 27 19:31:47 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
@@ -47,6 +48,7 @@ public class MiniQJMHACluster {
public static class Builder {
private final Configuration conf;
+ private StartupOption startOpt = null;
private final MiniDFSCluster.Builder dfsBuilder;
public Builder(Configuration conf) {
@@ -61,6 +63,10 @@ public class MiniQJMHACluster {
public MiniQJMHACluster build() throws IOException {
return new MiniQJMHACluster(this);
}
+
+ public void startupOption(StartupOption startOpt) {
+ this.startOpt = startOpt;
+ }
}
public static MiniDFSNNTopology createDefaultTopology() {
@@ -95,6 +101,9 @@ public class MiniQJMHACluster {
Configuration confNN0 = cluster.getConfiguration(0);
NameNode.initializeSharedEdits(confNN0, true);
+ cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt);
+ cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt);
+
// restart the cluster
cluster.restartNameNodes();
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Mon Jan 27 19:31:47 2014
@@ -27,6 +27,8 @@ import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.junit.Test;
@@ -191,6 +193,29 @@ public class TestGenericJournalConf {
shouldPromptCalled = true;
return false;
}
+
+ @Override
+ public void doPreUpgrade() throws IOException {}
+
+ @Override
+ public void doUpgrade(Storage storage) throws IOException {}
+
+ @Override
+ public void doFinalize() throws IOException {}
+
+ @Override
+ public boolean canRollBack(StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public void doRollback() throws IOException {}
+
+ @Override
+ public long getJournalCTime() throws IOException {
+ return -1;
+ }
}
public static class BadConstructorJournalManager extends DummyJournalManager {
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java Mon Jan 27 19:31:47 2014
@@ -91,7 +91,7 @@ public class TestBootstrapStandby {
fail("Did not throw");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
- "Cannot start an HA namenode with name dirs that need recovery",
+ "storage directory does not exist or is not accessible",
ioe);
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java Mon Jan 27 19:31:47 2014
@@ -1,89 +1,506 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hdfs.server.namenode.ha;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster.Builder;
+import org.apache.hadoop.hdfs.qjournal.server.Journal;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.hdfs.util.PersistentLongFile;
import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Lists;
+import com.google.common.base.Joiner;
/**
* Tests for upgrading with HA enabled.
*/
public class TestDFSUpgradeWithHA {
-
+
private static final Log LOG = LogFactory.getLog(TestDFSUpgradeWithHA.class);
+
+ private Configuration conf;
+
+ @Before
+ public void createConfiguration() {
+ conf = new HdfsConfiguration();
+ // Turn off persistent IPC, so that the DFSClient can survive NN restart
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
+ 0);
+ }
+
+ private static void assertCTimesEqual(MiniDFSCluster cluster) {
+ long nn1CTime = cluster.getNamesystem(0).getFSImage().getStorage().getCTime();
+ long nn2CTime = cluster.getNamesystem(1).getFSImage().getStorage().getCTime();
+ assertEquals(nn1CTime, nn2CTime);
+ }
+
+ private static void checkClusterPreviousDirExistence(MiniDFSCluster cluster,
+ boolean shouldExist) {
+ for (int i = 0; i < 2; i++) {
+ checkNnPreviousDirExistence(cluster, i, shouldExist);
+ }
+ }
+
+ private static void checkNnPreviousDirExistence(MiniDFSCluster cluster,
+ int index, boolean shouldExist) {
+ Collection<URI> nameDirs = cluster.getNameDirs(index);
+ for (URI nnDir : nameDirs) {
+ checkPreviousDirExistence(new File(nnDir), shouldExist);
+ }
+ }
+ private static void checkJnPreviousDirExistence(MiniQJMHACluster jnCluster,
+ boolean shouldExist) throws IOException {
+ for (int i = 0; i < 3; i++) {
+ checkPreviousDirExistence(
+ jnCluster.getJournalCluster().getJournalDir(i, "ns1"), shouldExist);
+ }
+ if (shouldExist) {
+ assertEpochFilesCopied(jnCluster);
+ }
+ }
+
+ private static void assertEpochFilesCopied(MiniQJMHACluster jnCluster)
+ throws IOException {
+ for (int i = 0; i < 3; i++) {
+ File journalDir = jnCluster.getJournalCluster().getJournalDir(i, "ns1");
+ File currDir = new File(journalDir, "current");
+ File prevDir = new File(journalDir, "previous");
+ for (String fileName : new String[]{ Journal.LAST_PROMISED_FILENAME,
+ Journal.LAST_WRITER_EPOCH }) {
+ File prevFile = new File(prevDir, fileName);
+ // Possible the prev file doesn't exist, e.g. if there has never been a
+ // writer before the upgrade.
+ if (prevFile.exists()) {
+ PersistentLongFile prevLongFile = new PersistentLongFile(prevFile, -10);
+ PersistentLongFile currLongFile = new PersistentLongFile(new File(currDir,
+ fileName), -11);
+ assertTrue("Value in " + fileName + " has decreased on upgrade in "
+ + journalDir, prevLongFile.get() <= currLongFile.get());
+ }
+ }
+ }
+ }
+
+ private static void checkPreviousDirExistence(File rootDir,
+ boolean shouldExist) {
+ File previousDir = new File(rootDir, "previous");
+ if (shouldExist) {
+ assertTrue(previousDir + " does not exist", previousDir.exists());
+ } else {
+ assertFalse(previousDir + " does exist", previousDir.exists());
+ }
+ }
+
+ private void runFinalizeCommand(MiniDFSCluster cluster)
+ throws IOException {
+ HATestUtil.setFailoverConfigurations(cluster, conf);
+ new DFSAdmin(conf).finalizeUpgrade();
+ }
+
/**
- * Make sure that an HA NN refuses to start if given an upgrade-related
- * startup option.
+ * Ensure that an admin cannot finalize an HA upgrade without at least one NN
+ * being active.
*/
@Test
- public void testStartingWithUpgradeOptionsFails() throws IOException {
- for (StartupOption startOpt : Lists.newArrayList(new StartupOption[] {
- StartupOption.UPGRADE, StartupOption.FINALIZE,
- StartupOption.ROLLBACK })) {
- MiniDFSCluster cluster = null;
+ public void testCannotFinalizeIfNoActive() throws IOException,
+ URISyntaxException {
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(0)
+ .build();
+
+ File sharedDir = new File(cluster.getSharedEditsDir(0, 1));
+
+ // No upgrade is in progress at the moment.
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+ checkPreviousDirExistence(sharedDir, false);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkPreviousDirExistence(sharedDir, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Restart NN0 without the -upgrade flag, to make sure that works.
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
+ cluster.restartNameNode(0, false);
+
+ // Make sure we can still do FS ops after upgrading.
+ cluster.transitionToActive(0);
+ assertTrue(fs.mkdirs(new Path("/foo3")));
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ // Now restart NN1 and make sure that we can do ops against that as well.
+ cluster.restartNameNode(1);
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+ assertTrue(fs.mkdirs(new Path("/foo4")));
+
+ assertCTimesEqual(cluster);
+
+ // Now there's no active NN.
+ cluster.transitionToStandby(1);
+
try {
- cluster = new MiniDFSCluster.Builder(new Configuration())
- .nnTopology(MiniDFSNNTopology.simpleHATopology())
- .startupOption(startOpt)
- .numDataNodes(0)
- .build();
- fail("Should not have been able to start an HA NN in upgrade mode");
- } catch (IllegalArgumentException iae) {
+ runFinalizeCommand(cluster);
+ fail("Should not have been able to finalize upgrade with no NN active");
+ } catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
- "Cannot perform DFS upgrade with HA enabled.", iae);
- LOG.info("Got expected exception", iae);
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
+ "Cannot finalize with no NameNode active", ioe);
+ }
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
}
}
}
-
+
/**
- * Make sure that an HA NN won't start if a previous upgrade was in progress.
+ * Make sure that an HA NN with NFS-based HA can successfully start and
+ * upgrade.
*/
@Test
- public void testStartingWithUpgradeInProgressFails() throws Exception {
+ public void testNfsUpgrade() throws IOException, URISyntaxException {
MiniDFSCluster cluster = null;
+ FileSystem fs = null;
try {
- cluster = new MiniDFSCluster.Builder(new Configuration())
+ cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(0)
.build();
+ File sharedDir = new File(cluster.getSharedEditsDir(0, 1));
+
+ // No upgrade is in progress at the moment.
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+ checkPreviousDirExistence(sharedDir, false);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkPreviousDirExistence(sharedDir, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Restart NN0 without the -upgrade flag, to make sure that works.
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
+ cluster.restartNameNode(0, false);
+
+ // Make sure we can still do FS ops after upgrading.
+ cluster.transitionToActive(0);
+ assertTrue(fs.mkdirs(new Path("/foo3")));
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ // Now restart NN1 and make sure that we can do ops against that as well.
+ cluster.restartNameNode(1);
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+ assertTrue(fs.mkdirs(new Path("/foo4")));
+
+ assertCTimesEqual(cluster);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure that an HA NN can successfully upgrade when configured using
+ * JournalNodes.
+ */
+ @Test
+ public void testUpgradeWithJournalNodes() throws IOException,
+ URISyntaxException {
+ MiniQJMHACluster qjCluster = null;
+ FileSystem fs = null;
+ try {
+ Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder()
+ .numDataNodes(0);
+ qjCluster = builder.build();
+
+ MiniDFSCluster cluster = qjCluster.getDfsCluster();
+
+ // No upgrade is in progress at the moment.
+ checkJnPreviousDirExistence(qjCluster, false);
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkJnPreviousDirExistence(qjCluster, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Restart NN0 without the -upgrade flag, to make sure that works.
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
+ cluster.restartNameNode(0, false);
+
+ // Make sure we can still do FS ops after upgrading.
+ cluster.transitionToActive(0);
+ assertTrue(fs.mkdirs(new Path("/foo3")));
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ // Now restart NN1 and make sure that we can do ops against that as well.
+ cluster.restartNameNode(1);
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+ assertTrue(fs.mkdirs(new Path("/foo4")));
+
+ assertCTimesEqual(cluster);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (qjCluster != null) {
+ qjCluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testFinalizeWithJournalNodes() throws IOException,
+ URISyntaxException {
+ MiniQJMHACluster qjCluster = null;
+ FileSystem fs = null;
+ try {
+ Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder()
+ .numDataNodes(0);
+ qjCluster = builder.build();
+
+ MiniDFSCluster cluster = qjCluster.getDfsCluster();
+
+ // No upgrade is in progress at the moment.
+ checkJnPreviousDirExistence(qjCluster, false);
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkJnPreviousDirExistence(qjCluster, true);
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ cluster.restartNameNode(1);
+
+ runFinalizeCommand(cluster);
+
+ checkClusterPreviousDirExistence(cluster, false);
+ checkJnPreviousDirExistence(qjCluster, false);
+ assertCTimesEqual(cluster);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (qjCluster != null) {
+ qjCluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure that even if the NN which initiated the upgrade is in the standby
+ * state that we're allowed to finalize.
+ */
+ @Test
+ public void testFinalizeFromSecondNameNodeWithJournalNodes()
+ throws IOException, URISyntaxException {
+ MiniQJMHACluster qjCluster = null;
+ FileSystem fs = null;
+ try {
+ Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder()
+ .numDataNodes(0);
+ qjCluster = builder.build();
+
+ MiniDFSCluster cluster = qjCluster.getDfsCluster();
+
+ // No upgrade is in progress at the moment.
+ checkJnPreviousDirExistence(qjCluster, false);
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkJnPreviousDirExistence(qjCluster, true);
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ cluster.restartNameNode(1);
+
+ // Make the second NN (not the one that initiated the upgrade) active when
+ // the finalize command is run.
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+
+ runFinalizeCommand(cluster);
+
+ checkClusterPreviousDirExistence(cluster, false);
+ checkJnPreviousDirExistence(qjCluster, false);
+ assertCTimesEqual(cluster);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (qjCluster != null) {
+ qjCluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure that an HA NN will start if a previous upgrade was in progress.
+ */
+ @Test
+ public void testStartingWithUpgradeInProgressSucceeds() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(0)
+ .build();
+
// Simulate an upgrade having started.
for (int i = 0; i < 2; i++) {
for (URI uri : cluster.getNameDirs(i)) {
@@ -92,18 +509,226 @@ public class TestDFSUpgradeWithHA {
assertTrue(prevTmp.mkdirs());
}
}
-
+
cluster.restartNameNodes();
- fail("Should not have been able to start an HA NN with an in-progress upgrade");
- } catch (IOException ioe) {
- GenericTestUtils.assertExceptionContains(
- "Cannot start an HA namenode with name dirs that need recovery.",
- ioe);
- LOG.info("Got expected exception", ioe);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
+
+ /**
+ * Test rollback with NFS shared dir.
+ */
+ @Test
+ public void testRollbackWithNfs() throws Exception {
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(0)
+ .build();
+
+ File sharedDir = new File(cluster.getSharedEditsDir(0, 1));
+
+ // No upgrade is in progress at the moment.
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+ checkPreviousDirExistence(sharedDir, false);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkPreviousDirExistence(sharedDir, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ cluster.restartNameNode(1);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkPreviousDirExistence(sharedDir, true);
+ assertCTimesEqual(cluster);
+
+ // Now shut down the cluster and do the rollback.
+ Collection<URI> nn1NameDirs = cluster.getNameDirs(0);
+ cluster.shutdown();
+
+ conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs));
+ NameNode.doRollback(conf, false);
+
+ // The rollback operation should have rolled back the first NN's local
+ // dirs, and the shared dir, but not the other NN's dirs. Those have to be
+ // done by bootstrapping the standby.
+ checkNnPreviousDirExistence(cluster, 0, false);
+ checkPreviousDirExistence(sharedDir, false);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testRollbackWithJournalNodes() throws IOException,
+ URISyntaxException {
+ MiniQJMHACluster qjCluster = null;
+ FileSystem fs = null;
+ try {
+ Builder builder = new MiniQJMHACluster.Builder(conf);
+ builder.getDfsBuilder()
+ .numDataNodes(0);
+ qjCluster = builder.build();
+
+ MiniDFSCluster cluster = qjCluster.getDfsCluster();
+
+ // No upgrade is in progress at the moment.
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+ checkJnPreviousDirExistence(qjCluster, false);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkJnPreviousDirExistence(qjCluster, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Now bootstrap the standby with the upgraded info.
+ int rc = BootstrapStandby.run(
+ new String[]{"-force"},
+ cluster.getConfiguration(1));
+ assertEquals(0, rc);
+
+ cluster.restartNameNode(1);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkJnPreviousDirExistence(qjCluster, true);
+ assertCTimesEqual(cluster);
+
+ // Shut down the NNs, but deliberately leave the JNs up and running.
+ Collection<URI> nn1NameDirs = cluster.getNameDirs(0);
+ cluster.shutdown();
+
+ conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs));
+ NameNode.doRollback(conf, false);
+
+ // The rollback operation should have rolled back the first NN's local
+ // dirs, and the shared dir, but not the other NN's dirs. Those have to be
+ // done by bootstrapping the standby.
+ checkNnPreviousDirExistence(cluster, 0, false);
+ checkJnPreviousDirExistence(qjCluster, false);
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (qjCluster != null) {
+ qjCluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure that starting a second NN with the -upgrade flag fails if the
+ * other NN has already done that.
+ */
+ @Test
+ public void testCannotUpgradeSecondNameNode() throws IOException,
+ URISyntaxException {
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(0)
+ .build();
+
+ File sharedDir = new File(cluster.getSharedEditsDir(0, 1));
+
+ // No upgrade is in progress at the moment.
+ checkClusterPreviousDirExistence(cluster, false);
+ assertCTimesEqual(cluster);
+ checkPreviousDirExistence(sharedDir, false);
+
+ // Transition NN0 to active and do some FS ops.
+ cluster.transitionToActive(0);
+ fs = HATestUtil.configureFailoverFs(cluster, conf);
+ assertTrue(fs.mkdirs(new Path("/foo1")));
+
+ // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
+ // flag.
+ cluster.shutdownNameNode(1);
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
+ cluster.restartNameNode(0, false);
+
+ checkNnPreviousDirExistence(cluster, 0, true);
+ checkNnPreviousDirExistence(cluster, 1, false);
+ checkPreviousDirExistence(sharedDir, true);
+
+ // NN0 should come up in the active state when given the -upgrade option,
+ // so no need to transition it to active.
+ assertTrue(fs.mkdirs(new Path("/foo2")));
+
+ // Restart NN0 without the -upgrade flag, to make sure that works.
+ cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
+ cluster.restartNameNode(0, false);
+
+ // Make sure we can still do FS ops after upgrading.
+ cluster.transitionToActive(0);
+ assertTrue(fs.mkdirs(new Path("/foo3")));
+
+ // Make sure that starting the second NN with the -upgrade flag fails.
+ cluster.getNameNodeInfos()[1].setStartOpt(StartupOption.UPGRADE);
+ try {
+ cluster.restartNameNode(1, false);
+ fail("Should not have been able to start second NN with -upgrade");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "It looks like the shared log is already being upgraded", ioe);
+ }
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java Mon Jan 27 19:31:47 2014
@@ -96,7 +96,7 @@ public class TestInitializeSharedEdits {
} catch (IOException ioe) {
LOG.info("Got expected exception", ioe);
GenericTestUtils.assertExceptionContains(
- "Cannot start an HA namenode with name dirs that need recovery", ioe);
+ "storage directory does not exist or is not accessible", ioe);
}
try {
cluster.restartNameNode(1, false);
@@ -104,7 +104,7 @@ public class TestInitializeSharedEdits {
} catch (IOException ioe) {
LOG.info("Got expected exception", ioe);
GenericTestUtils.assertExceptionContains(
- "Cannot start an HA namenode with name dirs that need recovery", ioe);
+ "storage directory does not exist or is not accessible", ioe);
}
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java?rev=1561802&r1=1561801&r2=1561802&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java Mon Jan 27 19:31:47 2014
@@ -17,22 +17,26 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
-import java.util.Random;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
import org.apache.hadoop.fs.FileStatus;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.util.ToolRunner;
public class TestSnapshotFileLength {
@@ -112,4 +116,61 @@ public class TestSnapshotFileLength {
assertThat(bytesRead, is(BLOCKSIZE));
fis.close();
}
+
+ /**
+ * Adding as part of jira HDFS-5343
+ * Test for checking the cat command on snapshot path it
+ * cannot read a file beyond snapshot file length
+ * @throws Exception
+ */
+ @Test (timeout = 600000)
+ public void testSnapshotFileLengthWithCatCommand() throws Exception {
+
+ FSDataInputStream fis = null;
+ FileStatus fileStatus = null;
+
+ int bytesRead;
+ byte[] buffer = new byte[BLOCKSIZE * 8];
+
+ hdfs.mkdirs(sub);
+ Path file1 = new Path(sub, file1Name);
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, SEED);
+
+ hdfs.allowSnapshot(sub);
+ hdfs.createSnapshot(sub, snapshot1);
+
+ DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE);
+
+ // Make sure we can read the entire file via its non-snapshot path.
+ fileStatus = hdfs.getFileStatus(file1);
+ assertEquals(fileStatus.getLen(), BLOCKSIZE * 2);
+ fis = hdfs.open(file1);
+ bytesRead = fis.read(buffer, 0, buffer.length);
+ assertEquals(bytesRead, BLOCKSIZE * 2);
+ fis.close();
+
+ Path file1snap1 =
+ SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name);
+ fis = hdfs.open(file1snap1);
+ fileStatus = hdfs.getFileStatus(file1snap1);
+ assertEquals(fileStatus.getLen(), BLOCKSIZE);
+ // Make sure we can only read up to the snapshot length.
+ bytesRead = fis.read(buffer, 0, buffer.length);
+ assertEquals(bytesRead, BLOCKSIZE);
+ fis.close();
+
+ PrintStream psBackup = System.out;
+ ByteArrayOutputStream bao = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(bao));
+ System.setErr(new PrintStream(bao));
+ // Make sure we can cat the file upto to snapshot length
+ FsShell shell = new FsShell();
+ try{
+ ToolRunner.run(conf, shell, new String[] { "-cat",
+ "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
+ assertEquals(bao.size(), BLOCKSIZE);
+ }finally{
+ System.setOut(psBackup);
+ }
+ }
}