You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/12/19 01:36:26 UTC
[05/20] hadoop git commit: HDFS-9490. MiniDFSCluster should change
block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
(cherry picked from commit 0ac8fb4b336dd03a037092c9712b962e7ed8f852)
(cherry picked from commit 58714f3b7357d05604987e1db8f6734a7d2827d9)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/134195df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/134195df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/134195df
Branch: refs/heads/branch-2.8
Commit: 134195df55c6b8d69840f423d61c12245bb2aad5
Parents: 12772f5
Author: Lei Xu <le...@apache.org>
Authored: Fri Dec 4 10:24:55 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:42 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +++-----
.../hdfs/server/datanode/FsDatasetTestUtils.java | 9 +++++++++
.../fsdataset/impl/FsDatasetImplTestUtils.java | 14 ++++++++++++++
.../namenode/ha/TestPendingCorruptDnMessages.java | 2 +-
5 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index baaa58d..8c421ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -865,6 +865,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9552. Document types of permission checks performed for HDFS
operations. (cnauroth)
+ HDFS-9490. MiniDFSCluster should change block generation stamp via
+ FsDatasetTestUtils. (Tony Wu via lei)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 0f4816e..092d9f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2026,12 +2026,10 @@ public class MiniDFSCluster {
getMaterializedReplica(i, blk).truncateMeta(newSize);
}
- public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
+ public void changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
long newGenStamp) throws IOException {
- File blockFile = getBlockFile(dnIndex, blk);
- File metaFile = FsDatasetUtil.findMetaFile(blockFile);
- return metaFile.renameTo(new File(DatanodeUtil.getMetaName(
- blockFile.getAbsolutePath(), newGenStamp)));
+ getFsDatasetTestUtils(dnIndex)
+ .changeStoredGenerationStamp(blk, newGenStamp);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
index 07fb7ce..fd47705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
@@ -242,4 +242,13 @@ public interface FsDatasetTestUtils {
* Get the persistently stored generation stamp.
*/
long getStoredGenerationStamp(ExtendedBlock block) throws IOException;
+
+ /**
+ * Change the persistently stored generation stamp.
+ * @param block the block whose generation stamp will be changed
+ * @param newGenStamp the new generation stamp
+ * @throws IOException
+ */
+ void changeStoredGenerationStamp(ExtendedBlock block, long newGenStamp)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
index 8fce163..320ae9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.datanode.Replica;
@@ -47,6 +48,7 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.Random;
/**
@@ -363,4 +365,16 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils {
File[] files = FileUtil.listFiles(dir);
return FsDatasetUtil.getGenerationStampFromFile(files, f);
}
+
+ @Override
+ public void changeStoredGenerationStamp(
+ ExtendedBlock block, long newGenStamp) throws IOException {
+ File blockFile =
+ dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId());
+ File metaFile = FsDatasetUtil.findMetaFile(blockFile);
+ File newMetaFile = new File(
+ DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp));
+ Files.move(metaFile.toPath(), newMetaFile.toPath(),
+ StandardCopyOption.ATOMIC_MOVE);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
index 443500c..5f116d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
@@ -68,7 +68,7 @@ public class TestPendingCorruptDnMessages {
// Change the gen stamp of the block on datanode to go back in time (gen
// stamps start at 1000)
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
- assertTrue(cluster.changeGenStampOfBlock(0, block, 900));
+ cluster.changeGenStampOfBlock(0, block, 900);
// Run directory dsscanner to update Datanode's volumeMap
DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));