You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/01/20 16:46:15 UTC

[1/4] hadoop git commit: HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a5ee4c09d -> 252bf56e5
  refs/heads/branch-2.7 2131b9820 -> eb3dcc073
  refs/heads/branch-2.8 15713f947 -> 3d38c1ef3
  refs/heads/trunk 57d0a9430 -> 142557869


HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14255786
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14255786
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14255786

Branch: refs/heads/trunk
Commit: 14255786908f991fd2022480fe5575533a3dc7ce
Parents: 57d0a94
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 20 21:04:21 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jan 20 21:04:21 2016 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt              |  3 +++
 .../server/datanode/fsdataset/impl/FsDatasetImpl.java    | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14255786/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b051f2f..e6074bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2646,6 +2646,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9648. TestStartup.testImageChecksum is broken by HDFS-9569's message
     change. (Wei-Chiu Chuang via Yongjun Zhang)
 
+    HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
+    createRbw (ade via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14255786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 5350974..cbd4123 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -936,8 +936,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
           + replicaInfo.getVolume().getStorageType());
     }
 
-    try (FsVolumeReference volumeRef = volumes.getNextVolume(
-        targetStorageType, block.getNumBytes())) {
+    FsVolumeReference volumeRef = null;
+    synchronized (this) {
+      volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+    }
+    try {
       File oldBlockFile = replicaInfo.getBlockFile();
       File oldMetaFile = replicaInfo.getMetaFile();
       FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
@@ -956,6 +959,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
       removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
           oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+    } finally {
+      if (volumeRef != null) {
+        volumeRef.close();
+      }
     }
 
     // Replace the old block if any to reschedule the scanning.


[4/4] hadoop git commit: HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

Posted by vi...@apache.org.
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

(cherry picked from commit 14255786908f991fd2022480fe5575533a3dc7ce)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb3dcc07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb3dcc07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb3dcc07

Branch: refs/heads/branch-2.7
Commit: eb3dcc07396616cf760088334b67fd06bf27a49a
Parents: 2131b98
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 20 21:04:21 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jan 20 21:11:39 2016 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt              |  3 +++
 .../server/datanode/fsdataset/impl/FsDatasetImpl.java    | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb3dcc07/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 073a0e1..012080e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -59,6 +59,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9648. TestStartup.testImageChecksum is broken by HDFS-9569's message
     change. (Wei-Chiu Chuang via Yongjun Zhang)
 
+    HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
+    createRbw (ade via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb3dcc07/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 3228629..f4e10f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -896,8 +896,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
           + replicaInfo.getVolume().getStorageType());
     }
 
-    try (FsVolumeReference volumeRef = volumes.getNextVolume(
-        targetStorageType, block.getNumBytes())) {
+    FsVolumeReference volumeRef = null;
+    synchronized (this) {
+      volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+    }
+    try {
       File oldBlockFile = replicaInfo.getBlockFile();
       File oldMetaFile = replicaInfo.getMetaFile();
       FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
@@ -916,6 +919,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
       removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
           oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+    } finally {
+      if (volumeRef != null) {
+        volumeRef.close();
+      }
     }
 
     // Replace the old block if any to reschedule the scanning.


[3/4] hadoop git commit: HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

Posted by vi...@apache.org.
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

(cherry picked from commit 14255786908f991fd2022480fe5575533a3dc7ce)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d38c1ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d38c1ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d38c1ef

Branch: refs/heads/branch-2.8
Commit: 3d38c1ef3ad31362b0be638fb0225b4b8c5b3d96
Parents: 15713f9
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 20 21:04:21 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jan 20 21:09:08 2016 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt              |  3 +++
 .../server/datanode/fsdataset/impl/FsDatasetImpl.java    | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d38c1ef/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 04a323f..7ef5a74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1712,6 +1712,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9648. TestStartup.testImageChecksum is broken by HDFS-9569's message
     change. (Wei-Chiu Chuang via Yongjun Zhang)
 
+    HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
+    createRbw (ade via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d38c1ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 544169d..950462a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -939,8 +939,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
           + replicaInfo.getVolume().getStorageType());
     }
 
-    try (FsVolumeReference volumeRef = volumes.getNextVolume(
-        targetStorageType, block.getNumBytes())) {
+    FsVolumeReference volumeRef = null;
+    synchronized (this) {
+      volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+    }
+    try {
       File oldBlockFile = replicaInfo.getBlockFile();
       File oldMetaFile = replicaInfo.getMetaFile();
       FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
@@ -959,6 +962,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
       removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
           oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+    } finally {
+      if (volumeRef != null) {
+        volumeRef.close();
+      }
     }
 
     // Replace the old block if any to reschedule the scanning.


[2/4] hadoop git commit: HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

Posted by vi...@apache.org.
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)

(cherry picked from commit 14255786908f991fd2022480fe5575533a3dc7ce)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/252bf56e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/252bf56e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/252bf56e

Branch: refs/heads/branch-2
Commit: 252bf56e59cb0bcfed87b32c16cb8d2a445784b9
Parents: a5ee4c0
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 20 21:04:21 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jan 20 21:05:44 2016 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt              |  3 +++
 .../server/datanode/fsdataset/impl/FsDatasetImpl.java    | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/252bf56e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 62186bf..4544960 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1765,6 +1765,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9648. TestStartup.testImageChecksum is broken by HDFS-9569's message
     change. (Wei-Chiu Chuang via Yongjun Zhang)
 
+    HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
+    createRbw (ade via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/252bf56e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index dd206c7..e77c9a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -941,8 +941,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
           + replicaInfo.getVolume().getStorageType());
     }
 
-    try (FsVolumeReference volumeRef = volumes.getNextVolume(
-        targetStorageType, block.getNumBytes())) {
+    FsVolumeReference volumeRef = null;
+    synchronized (this) {
+      volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+    }
+    try {
       File oldBlockFile = replicaInfo.getBlockFile();
       File oldMetaFile = replicaInfo.getMetaFile();
       FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
@@ -961,6 +964,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
       removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
           oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+    } finally {
+      if (volumeRef != null) {
+        volumeRef.close();
+      }
     }
 
     // Replace the old block if any to reschedule the scanning.