You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2015/02/20 16:09:29 UTC
hadoop git commit: HDFS-7788. Post-2.6 namenode may not start up with
an image containing inodes created with an old release. Contributed by
Rushabh Shah.
Repository: hadoop
Updated Branches:
refs/heads/trunk b9a17909b -> 7ae5255a1
HDFS-7788. Post-2.6 namenode may not start up with an image containing inodes created with an old release. Contributed by Rushabh Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ae5255a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ae5255a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ae5255a
Branch: refs/heads/trunk
Commit: 7ae5255a1613ccfb43646f33eabacf1062c86e93
Parents: b9a1790
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Feb 20 09:06:07 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Feb 20 09:09:02 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 ++
.../apache/hadoop/hdfs/util/LongBitFormat.java | 4 ++
.../hdfs/server/namenode/TestFSImage.java | 48 +++++++++++++++++++
.../resources/image-with-zero-block-size.tar.gz | Bin 0 -> 1452 bytes
5 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ae5255a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5f3cc02..71ce48f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -992,6 +992,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7808. Remove obsolete -ns options in in DFSHAAdmin.java.
(Arshad Mohammad via wheat9)
+ HDFS-7788. Post-2.6 namenode may not start up with an image containing
+ inodes created with an old release. (Rushabh Shah via kihwal)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ae5255a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 24e25ec..3743bf0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -107,6 +107,9 @@ public class INodeFile extends INodeWithAdditionalFields
static long toLong(long preferredBlockSize, short replication,
byte storagePolicyID) {
long h = 0;
+ if (preferredBlockSize == 0) {
+ preferredBlockSize = PREFERRED_BLOCK_SIZE.BITS.getMin();
+ }
h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
h = REPLICATION.BITS.combine(replication, h);
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ae5255a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
index 863d9f7..9399d84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
@@ -64,4 +64,8 @@ public class LongBitFormat implements Serializable {
}
return (record & ~MASK) | (value << OFFSET);
}
+
+ public long getMin() {
+ return MIN;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ae5255a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index c68ae04..f7dad18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -28,10 +28,13 @@ import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -40,10 +43,14 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
public class TestFSImage {
+ private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
+ "image-with-zero-block-size.tar.gz";
@Test
public void testPersist() throws IOException {
Configuration conf = new Configuration();
@@ -183,4 +190,45 @@ public class TestFSImage {
}
}
}
+
+ /**
+ * In this test case, I have created an image with a file having
+ * preferredblockSize = 0. We are trying to read this image (since file with
+ * preferredblockSize = 0 was allowed pre 2.1.0-beta version. The namenode
+ * after 2.6 version will not be able to read this particular file.
+ * See HDFS-7788 for more information.
+ * @throws Exception
+ */
+ @Test
+ public void testZeroBlockSize() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ String tarFile = System.getProperty("test.cache.data", "build/test/cache")
+ + "/" + HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ;
+ String testDir = PathUtils.getTestDirName(getClass());
+ File dfsDir = new File(testDir, "image-with-zero-block-size");
+ if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
+ throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
+ }
+ FileUtil.unTar(new File(tarFile), new File(testDir));
+ File nameDir = new File(dfsDir, "name");
+ GenericTestUtils.assertExists(nameDir);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ nameDir.getAbsolutePath());
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .format(false)
+ .manageDataDfsDirs(false)
+ .manageNameDfsDirs(false)
+ .waitSafeMode(false)
+ .build();
+ try {
+ FileSystem fs = cluster.getFileSystem();
+ Path testPath = new Path("/tmp/zeroBlockFile");
+ assertTrue("File /tmp/zeroBlockFile doesn't exist ", fs.exists(testPath));
+ assertTrue("Name node didn't come up", cluster.isNameNodeUp(0));
+ } finally {
+ cluster.shutdown();
+ //Clean up
+ FileUtil.fullyDelete(dfsDir);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ae5255a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
new file mode 100644
index 0000000..8bfc454
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz differ