You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2015/12/06 08:13:31 UTC
[29/38] hadoop git commit: HDFS-9490. MiniDFSCluster should change
block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ac8fb4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ac8fb4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ac8fb4b
Branch: refs/heads/yarn-2877
Commit: 0ac8fb4b336dd03a037092c9712b962e7ed8f852
Parents: 3fa33b5
Author: Lei Xu <le...@apache.org>
Authored: Fri Dec 4 10:24:55 2015 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Fri Dec 4 10:24:55 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +++-----
.../hdfs/server/datanode/FsDatasetTestUtils.java | 9 +++++++++
.../fsdataset/impl/FsDatasetImplTestUtils.java | 14 ++++++++++++++
.../namenode/ha/TestPendingCorruptDnMessages.java | 2 +-
5 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 17cbe29..89094bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1710,6 +1710,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9485. Make BlockManager#removeFromExcessReplicateMap accept BlockInfo
instead of Block. (Mingliang Liu via jing9)
+ HDFS-9490. MiniDFSCluster should change block generation stamp via
+ FsDatasetTestUtils. (Tony Wu via lei)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 6baea25..3e25177 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2141,12 +2141,10 @@ public class MiniDFSCluster {
getMaterializedReplica(i, blk).truncateMeta(newSize);
}
- public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
+ public void changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
long newGenStamp) throws IOException {
- File blockFile = getBlockFile(dnIndex, blk);
- File metaFile = FsDatasetUtil.findMetaFile(blockFile);
- return metaFile.renameTo(new File(DatanodeUtil.getMetaName(
- blockFile.getAbsolutePath(), newGenStamp)));
+ getFsDatasetTestUtils(dnIndex)
+ .changeStoredGenerationStamp(blk, newGenStamp);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
index 07fb7ce..fd47705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
@@ -242,4 +242,13 @@ public interface FsDatasetTestUtils {
* Get the persistently stored generation stamp.
*/
long getStoredGenerationStamp(ExtendedBlock block) throws IOException;
+
+ /**
+ * Change the persistently stored generation stamp.
+ * @param block the block whose generation stamp will be changed
+ * @param newGenStamp the new generation stamp
+ * @throws IOException
+ */
+ void changeStoredGenerationStamp(ExtendedBlock block, long newGenStamp)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
index 8fce163..320ae9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.datanode.Replica;
@@ -47,6 +48,7 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.Random;
/**
@@ -363,4 +365,16 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils {
File[] files = FileUtil.listFiles(dir);
return FsDatasetUtil.getGenerationStampFromFile(files, f);
}
+
+ @Override
+ public void changeStoredGenerationStamp(
+ ExtendedBlock block, long newGenStamp) throws IOException {
+ File blockFile =
+ dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId());
+ File metaFile = FsDatasetUtil.findMetaFile(blockFile);
+ File newMetaFile = new File(
+ DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp));
+ Files.move(metaFile.toPath(), newMetaFile.toPath(),
+ StandardCopyOption.ATOMIC_MOVE);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
index 443500c..5f116d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
@@ -68,7 +68,7 @@ public class TestPendingCorruptDnMessages {
// Change the gen stamp of the block on datanode to go back in time (gen
// stamps start at 1000)
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
- assertTrue(cluster.changeGenStampOfBlock(0, block, 900));
+ cluster.changeGenStampOfBlock(0, block, 900);
// Run directory dsscanner to update Datanode's volumeMap
DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));