You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2016/09/13 08:16:42 UTC
[18/32] hadoop git commit: Addendum patch for HDFS-9781.
FsDatasetImpl#getBlockReports can occasionally throw NullPointerException.
Contributed by Manoj Govindassamy.
Addendum patch for HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0b03836
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0b03836
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0b03836
Branch: refs/heads/HADOOP-12756
Commit: a0b0383677c037d4492907e5f119eb0e72390faf
Parents: d4d0768
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Sep 9 10:52:05 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Sep 9 10:54:14 2016 -0700
----------------------------------------------------------------------
.../fsdataset/impl/TestFsDatasetImpl.java | 83 +++++++++++++-------
1 file changed, 54 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b03836/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index b946803..b3f04d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -589,20 +589,21 @@ public class TestFsDatasetImpl {
// Will write and remove on dn0.
final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0);
final CountDownLatch startFinalizeLatch = new CountDownLatch(1);
- final CountDownLatch brReceivedLatch = new CountDownLatch(1);
- final CountDownLatch volRemovedLatch = new CountDownLatch(1);
+ final CountDownLatch blockReportReceivedLatch = new CountDownLatch(1);
+ final CountDownLatch volRemoveStartedLatch = new CountDownLatch(1);
+ final CountDownLatch volRemoveCompletedLatch = new CountDownLatch(1);
class BlockReportThread extends Thread {
public void run() {
// Lets wait for the volume remove process to start
try {
- volRemovedLatch.await();
+ volRemoveStartedLatch.await();
} catch (Exception e) {
LOG.info("Unexpected exception when waiting for vol removal:", e);
}
LOG.info("Getting block report");
dataset.getBlockReports(eb.getBlockPoolId());
LOG.info("Successfully received block report");
- brReceivedLatch.countDown();
+ blockReportReceivedLatch.countDown();
}
}
@@ -623,7 +624,7 @@ public class TestFsDatasetImpl {
}
// Lets wait for the other thread finish getting block report
- brReceivedLatch.await();
+ blockReportReceivedLatch.await();
dataset.finalizeBlock(eb);
LOG.info("FinalizeBlock finished");
@@ -633,34 +634,58 @@ public class TestFsDatasetImpl {
}
}
- ResponderThread res = new ResponderThread();
- res.start();
+ class VolRemoveThread extends Thread {
+ public void run() {
+ try {
+ Set<File> volumesToRemove = new HashSet<>();
+ volumesToRemove.add(StorageLocation.parse(
+ dataset.getVolume(eb).getBasePath()).getFile());
+ /**
+ * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
+ * IllegalMonitorStateException when there is a parallel reader/writer
+ * to the volume. Remove below exception handling block after fixing
+ * HDFS-10830.
+ */
+ LOG.info("Removing volume " + volumesToRemove);
+ dataset.removeVolumes(volumesToRemove, true);
+ volRemoveCompletedLatch.countDown();
+ LOG.info("Removed volume " + volumesToRemove);
+ } catch (Exception e) {
+ LOG.info("Unexpected issue while removing volume: ", e);
+ volRemoveCompletedLatch.countDown();
+ }
+ }
+ }
+
+ // Start the volume write operation
+ ResponderThread responderThread = new ResponderThread();
+ responderThread.start();
startFinalizeLatch.await();
- // Verify if block report can be received
- // when volume is being removed
- final BlockReportThread brt = new BlockReportThread();
- brt.start();
+ // Start the block report get operation
+ final BlockReportThread blockReportThread = new BlockReportThread();
+ blockReportThread.start();
- Set<File> volumesToRemove = new HashSet<>();
- volumesToRemove.add(
- StorageLocation.parse(dataset.getVolume(eb).getBasePath()).getFile());
- /**
- * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
- * IllegalMonitorStateException when there is a parallel reader/writer
- * to the volume. Remove below try/catch block after fixing HDFS-10830.
- */
- try {
- LOG.info("Removing volume " + volumesToRemove);
- dataset.removeVolumes(volumesToRemove, true);
- } catch (Exception e) {
- LOG.info("Unexpected issue while removing volume: ", e);
- } finally {
- volRemovedLatch.countDown();
- }
+ // Start the volume remove operation
+ VolRemoveThread volRemoveThread = new VolRemoveThread();
+ volRemoveThread.start();
+
+ // Let volume write and remove operation be
+ // blocked for few seconds
+ Thread.sleep(2000);
+
+ // Signal block report receiver and volume writer
+ // thread to complete their operations so that vol
+ // remove can proceed
+ volRemoveStartedLatch.countDown();
+
+ // Verify if block report can be received
+ // when volume is in use and also being removed
+ blockReportReceivedLatch.await();
- LOG.info("Volumes removed");
- brReceivedLatch.await();
+ // Verify if volume can be removed safely when there
+ // are read/write operation in-progress
+ volRemoveCompletedLatch.await();
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org