You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2018/09/25 09:30:08 UTC
hadoop git commit: HDFS-12319. DirectoryScanner will throw
IllegalStateException when Multiple BP's are present. Contributed by Brahma
Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 b9b737732 -> bd79bfc3a
HDFS-12319. DirectoryScanner will throw IllegalStateException when Multiple BP's are present. Contributed by Brahma Reddy Battula.
(cherry picked from commit 2cae387402392f9dadadc95b9bc1677d93515758)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd79bfc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd79bfc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd79bfc3
Branch: refs/heads/branch-2.7
Commit: bd79bfc3a08ee4e3fd552ecc88c288fbb4098f26
Parents: b9b7377
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Aug 25 10:41:52 2017 -0700
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Sep 25 18:29:08 2018 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 15 +++++
.../hdfs/server/datanode/DirectoryScanner.java | 5 +-
.../server/datanode/TestDirectoryScanner.java | 65 ++++++++++++++++++--
3 files changed, 78 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd79bfc3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6d2d2c8..a59f88e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,5 +1,20 @@
Hadoop HDFS Change Log
+Release 2.7.8 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ HDFS-12319. DirectoryScanner will throw IllegalStateException
+ when Multiple BP's are present. Contributed by Brahma Reddy Battula.
+
Release 2.7.7 - 2018-07-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd79bfc3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index d1a44d8..d06c51b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -818,13 +818,12 @@ public class DirectoryScanner implements Runnable {
public ScanInfoPerBlockPool call() throws IOException {
String[] bpList = volume.getBlockPoolList();
ScanInfoPerBlockPool result = new ScanInfoPerBlockPool(bpList.length);
+ perfTimer.start();
+ throttleTimer.start();
for (String bpid : bpList) {
LinkedList<ScanInfo> report = new LinkedList<>();
File bpFinalizedDir = volume.getFinalizedDir(bpid);
- perfTimer.start();
- throttleTimer.start();
-
try {
result.put(bpid,
compileReport(volume, bpFinalizedDir, bpFinalizedDir, report));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd79bfc3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index 5046e97..cb23087 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -277,15 +278,22 @@ public class TestDirectoryScanner {
missingMemoryBlocks, mismatchBlocks, 0);
}
- private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile,
- long missingMemoryBlocks, long mismatchBlocks, long duplicateBlocks) throws IOException {
+ private void scan(long totalBlocks, int diffsize, long missingMetaFile,
+ long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks,
+ long duplicateBlocks) throws IOException {
scanner.reconcile();
-
+ verifyStats(totalBlocks, diffsize, missingMetaFile, missingBlockFile,
+ missingMemoryBlocks, mismatchBlocks, duplicateBlocks);
+ }
+
+ private void verifyStats(long totalBlocks, int diffsize, long missingMetaFile,
+ long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks,
+ long duplicateBlocks) {
assertTrue(scanner.diffs.containsKey(bpid));
LinkedList<DirectoryScanner.ScanInfo> diff = scanner.diffs.get(bpid);
assertTrue(scanner.stats.containsKey(bpid));
DirectoryScanner.Stats stats = scanner.stats.get(bpid);
-
+
assertEquals(diffsize, diff.size());
assertEquals(totalBlocks, stats.totalBlocks);
assertEquals(missingMetaFile, stats.missingMetaFile);
@@ -904,4 +912,53 @@ public class TestDirectoryScanner {
new File(TEST_VOLUME.getFinalizedDir(BPID_2).getAbsolutePath(),
"blk_567__1004.meta"));
}
+
+ @Test
+ public void testDirectoryScannerInFederatedCluster() throws Exception {
+ //Create Federated cluster with two nameservices and one DN
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(CONF)
+ .nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(2))
+ .numDataNodes(1).build();
+ cluster.waitActive();
+ cluster.transitionToActive(1);
+ cluster.transitionToActive(3);
+ DataNode dataNode = cluster.getDataNodes().get(0);
+ fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
+ //Create one block in first nameservice
+ FileSystem fs = cluster.getFileSystem(1);
+ int bp1Files = 1;
+ writeFile(fs, bp1Files);
+ //Create two blocks in second nameservice
+ FileSystem fs2 = cluster.getFileSystem(3);
+ int bp2Files = 2;
+ writeFile(fs2, bp2Files);
+ //Call the Directory scanner
+ scanner = new DirectoryScanner(dataNode, fds, CONF);
+ scanner.setRetainDiffs(true);
+ scanner.reconcile();
+ //Check blocks in corresponding BP
+ bpid = cluster.getNamesystem(1).getBlockPoolId();
+ verifyStats(bp1Files, 0, 0, 0, 0, 0, 0);
+ bpid = cluster.getNamesystem(3).getBlockPoolId();
+ verifyStats(bp2Files, 0, 0, 0, 0, 0, 0);
+ } finally {
+ if (scanner != null) {
+ scanner.shutdown();
+ scanner = null;
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private void writeFile(FileSystem fs, int numFiles) throws IOException {
+ final String fileName = "/" + GenericTestUtils.getMethodName();
+ final Path filePath = new Path(fileName);
+ for (int i = 0; i < numFiles; i++) {
+ DFSTestUtil.createFile(fs, filePath, 1, (short) 1, 0);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org