You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/03/13 02:11:30 UTC
[hadoop] branch branch-3.2 updated: HDFS-14081. hdfs dfsadmin
-metasave metasave_test results NPE. Contributed by Shweta Yakkali.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 1ceefa7 HDFS-14081. hdfs dfsadmin -metasave metasave_test results NPE. Contributed by Shweta Yakkali.
1ceefa7 is described below
commit 1ceefa726e1b531ec92c2ee2212b25c327644ef6
Author: Shweta Yakkali <sh...@cloudera.com>
AuthorDate: Wed Feb 20 14:28:37 2019 -0800
HDFS-14081. hdfs dfsadmin -metasave metasave_test results NPE. Contributed by Shweta Yakkali.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit 1bea785020a538115b3e08f41ff88167033d2775)
---
.../hdfs/server/blockmanagement/BlockManager.java | 13 +++++---
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 4 +--
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 14 +++++++--
.../server/blockmanagement/TestBlockManager.java | 35 ++++++++++++++++++++++
.../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 12 ++++++--
5 files changed, 67 insertions(+), 11 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index ebbba3b..8eb7670 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -838,8 +838,13 @@ public class BlockManager implements BlockStatsMXBean {
new ArrayList<DatanodeStorageInfo>();
NumberReplicas numReplicas = new NumberReplicas();
+ BlockInfo blockInfo = getStoredBlock(block);
+ if (blockInfo == null) {
+ out.println("Block "+ block + " is Null");
+ return;
+ }
// source node returned is not used
- chooseSourceDatanodes(getStoredBlock(block), containingNodes,
+ chooseSourceDatanodes(blockInfo, containingNodes,
containingLiveReplicasNodes, numReplicas,
new LinkedList<Byte>(), LowRedundancyBlocks.LEVEL);
@@ -848,7 +853,7 @@ public class BlockManager implements BlockStatsMXBean {
assert containingLiveReplicasNodes.size() >= numReplicas.liveReplicas();
int usableReplicas = numReplicas.liveReplicas() +
numReplicas.decommissionedAndDecommissioning();
-
+
if (block instanceof BlockInfo) {
BlockCollection bc = getBlockCollection((BlockInfo)block);
String fileName = (bc == null) ? "[orphaned]" : bc.getName();
@@ -1764,8 +1769,8 @@ public class BlockManager implements BlockStatsMXBean {
this.shouldPostponeBlocksFromFuture = postpone;
}
-
- private void postponeBlock(Block blk) {
+ @VisibleForTesting
+ void postponeBlock(Block blk) {
postponedMisreplicatedBlocks.add(blk);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 14e90d1..7c635ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1756,10 +1756,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void metaSave(String filename) throws IOException {
String operationName = "metaSave";
checkSuperuserPrivilege(operationName);
- checkOperation(OperationCategory.UNCHECKED);
+ checkOperation(OperationCategory.READ);
writeLock();
try {
- checkOperation(OperationCategory.UNCHECKED);
+ checkOperation(OperationCategory.READ);
File file = new File(System.getProperty("hadoop.log.dir"), filename);
PrintWriter out = new PrintWriter(new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8)));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index aa67e72..d1f8362 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolClientSideTranslatorPB;
import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.SecurityUtil;
@@ -1537,11 +1538,20 @@ public class DFSAdmin extends FsShell {
nsId, ClientProtocol.class);
List<IOException> exceptions = new ArrayList<>();
for (ProxyAndInfo<ClientProtocol> proxy : proxies) {
- try{
+ try {
proxy.getProxy().metaSave(pathname);
System.out.println("Created metasave file " + pathname
+ " in the log directory of namenode " + proxy.getAddress());
- } catch (IOException ioe){
+ } catch (RemoteException re) {
+ Exception unwrapped = re.unwrapRemoteException(
+ StandbyException.class);
+ if (unwrapped instanceof StandbyException) {
+ System.out.println("Skip Standby NameNode, since it cannot perform"
+ + " metasave operation");
+ } else {
+ throw re;
+ }
+ } catch (IOException ioe) {
System.out.println("Created metasave file " + pathname
+ " in the log directory of namenode " + proxy.getAddress()
+ " failed");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 0097da8..3bd41e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1478,6 +1478,41 @@ public class TestBlockManager {
}
}
+ /**
+ * Unit test to check the race condition for adding a Block to
+ * postponedMisreplicatedBlocks set which may not present in BlockManager
+ * thus avoiding NullPointerException.
+ **/
+ @Test
+ public void testMetaSavePostponedMisreplicatedBlocks() throws IOException {
+ bm.postponeBlock(new Block());
+
+ File file = new File("test.log");
+ PrintWriter out = new PrintWriter(file);
+
+ bm.metaSave(out);
+ out.flush();
+
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ }
+ String output = buffer.toString();
+ assertTrue("Metasave output should not have null block ",
+ output.contains("Block blk_0_0 is Null"));
+
+ } finally {
+ reader.close();
+ file.delete();
+ }
+ }
+
@Test
public void testMetaSaveMissingReplicas() throws Exception {
List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
index b85a8d8..627ea07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
@@ -414,16 +414,21 @@ public class TestDFSAdminWithHA {
@Test (timeout = 30000)
public void testMetaSave() throws Exception {
setUpHaCluster(false);
+ cluster.getDfsCluster().transitionToActive(0);
int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
assertEquals(err.toString().trim(), 0, exitCode);
- String message = "Created metasave file dfs.meta in the log directory"
- + " of namenode.*";
- assertOutputMatches(message + newLine + message + newLine);
+ String messageFromActiveNN = "Created metasave file dfs.meta "
+ + "in the log directory of namenode.*";
+ String messageFromStandbyNN = "Skip Standby NameNode, since it "
+ + "cannot perform metasave operation";
+ assertOutputMatches(messageFromActiveNN + newLine +
+ messageFromStandbyNN + newLine);
}
@Test (timeout = 30000)
public void testMetaSaveNN1UpNN2Down() throws Exception {
setUpHaCluster(false);
+ cluster.getDfsCluster().transitionToActive(0);
cluster.getDfsCluster().shutdownNameNode(1);
int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
assertNotEquals(err.toString().trim(), 0, exitCode);
@@ -437,6 +442,7 @@ public class TestDFSAdminWithHA {
@Test (timeout = 30000)
public void testMetaSaveNN1DownNN2Up() throws Exception {
setUpHaCluster(false);
+ cluster.getDfsCluster().transitionToActive(1);
cluster.getDfsCluster().shutdownNameNode(0);
int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
assertNotEquals(err.toString().trim(), 0, exitCode);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org