You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/01/07 09:51:55 UTC
[1/3] hadoop git commit: HDFS-9619. SimulatedFSDataset sometimes can
not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aa710bd46 -> 08f39bd5d
refs/heads/branch-2.8 a14baa05b -> 3303326dd
refs/heads/trunk 34cd7cd76 -> 6702e7d66
HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6702e7d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6702e7d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6702e7d6
Branch: refs/heads/trunk
Commit: 6702e7d66a4949a40bcdbd9f51ca1e746c8991e1
Parents: 34cd7cd
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 14:20:18 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 14:20:18 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/datanode/SimulatedFSDataset.java | 3 +-
.../server/datanode/TestSimulatedFSDataset.java | 47 ++++++++++++++++++++
3 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dfa9701..c451ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2567,6 +2567,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI.
(Archana T via wheat9)
+ HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the
+ correct namenode (Wei-Chiu Chuang via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 4316854..7a10379 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -521,7 +522,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
private final Map<String, Map<Block, BInfo>> blockMap
- = new HashMap<String, Map<Block,BInfo>>();
+ = new ConcurrentHashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
private final SimulatedVolume volume;
private final String datanodeUuid;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 8dc80d5..dd7d239 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
import org.apache.hadoop.util.DataChecksum;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -338,4 +340,49 @@ public class TestSimulatedFSDataset {
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
+
+ @Test
+ public void testConcurrentAddBlockPool() throws InterruptedException,
+ IOException {
+ final String[] bpids = {"BP-TEST1-", "BP-TEST2-"};
+ final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
+ class AddBlockPoolThread extends Thread {
+ private int id;
+ private IOException ioe;
+ public AddBlockPoolThread(int id) {
+ super();
+ this.id = id;
+ }
+ public void test() throws InterruptedException, IOException {
+ this.join();
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+ public void run() {
+ for (int i=0; i < 10000; i++) {
+ // add different block pools concurrently
+ String newbpid = bpids[id] + i;
+ fsdataset.addBlockPool(newbpid, conf);
+ // and then add a block into the pool
+ ExtendedBlock block = new ExtendedBlock(newbpid,1);
+ try {
+ // it will throw an exception if the block pool is not found
+ fsdataset.createTemporary(StorageType.DEFAULT, block);
+ } catch (IOException ioe) {
+ // JUnit does not capture exception in non-main thread,
+ // so cache it and then let main thread throw later.
+ this.ioe = ioe;
+ }
+ assert(fsdataset.getReplicaString(newbpid,1) != "null");
+ }
+ }
+ };
+ AddBlockPoolThread t1 = new AddBlockPoolThread(0);
+ AddBlockPoolThread t2 = new AddBlockPoolThread(1);
+ t1.start();
+ t2.start();
+ t1.test();
+ t2.test();
+ }
}
[3/3] hadoop git commit: HDFS-9619. SimulatedFSDataset sometimes can
not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Posted by vi...@apache.org.
HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
(cherry picked from commit 6702e7d66a4949a40bcdbd9f51ca1e746c8991e1)
(cherry picked from commit 08f39bd5d1b86db9a0efa84901408a615309bf48)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3303326d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3303326d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3303326d
Branch: refs/heads/branch-2.8
Commit: 3303326dd9ec33f3b921b3abfbf2098d3856d6d0
Parents: a14baa0
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 14:20:18 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 14:21:18 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/datanode/SimulatedFSDataset.java | 3 +-
.../server/datanode/TestSimulatedFSDataset.java | 47 ++++++++++++++++++++
3 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3303326d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3fdffb0..99ddff3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1637,6 +1637,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI.
(Archana T via wheat9)
+ HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the
+ correct namenode (Wei-Chiu Chuang via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3303326d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 4de1be7..17ca6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -522,7 +523,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
private final Map<String, Map<Block, BInfo>> blockMap
- = new HashMap<String, Map<Block,BInfo>>();
+ = new ConcurrentHashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
private final SimulatedVolume volume;
private final String datanodeUuid;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3303326d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 8dc80d5..dd7d239 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
import org.apache.hadoop.util.DataChecksum;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -338,4 +340,49 @@ public class TestSimulatedFSDataset {
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
+
+ @Test
+ public void testConcurrentAddBlockPool() throws InterruptedException,
+ IOException {
+ final String[] bpids = {"BP-TEST1-", "BP-TEST2-"};
+ final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
+ class AddBlockPoolThread extends Thread {
+ private int id;
+ private IOException ioe;
+ public AddBlockPoolThread(int id) {
+ super();
+ this.id = id;
+ }
+ public void test() throws InterruptedException, IOException {
+ this.join();
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+ public void run() {
+ for (int i=0; i < 10000; i++) {
+ // add different block pools concurrently
+ String newbpid = bpids[id] + i;
+ fsdataset.addBlockPool(newbpid, conf);
+ // and then add a block into the pool
+ ExtendedBlock block = new ExtendedBlock(newbpid,1);
+ try {
+ // it will throw an exception if the block pool is not found
+ fsdataset.createTemporary(StorageType.DEFAULT, block);
+ } catch (IOException ioe) {
+ // JUnit does not capture exception in non-main thread,
+ // so cache it and then let main thread throw later.
+ this.ioe = ioe;
+ }
+ assert(fsdataset.getReplicaString(newbpid,1) != "null");
+ }
+ }
+ };
+ AddBlockPoolThread t1 = new AddBlockPoolThread(0);
+ AddBlockPoolThread t2 = new AddBlockPoolThread(1);
+ t1.start();
+ t2.start();
+ t1.test();
+ t2.test();
+ }
}
[2/3] hadoop git commit: HDFS-9619. SimulatedFSDataset sometimes can
not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Posted by vi...@apache.org.
HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
(cherry picked from commit 6702e7d66a4949a40bcdbd9f51ca1e746c8991e1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08f39bd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08f39bd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08f39bd5
Branch: refs/heads/branch-2
Commit: 08f39bd5d1b86db9a0efa84901408a615309bf48
Parents: aa710bd
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 7 14:20:18 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 7 14:20:39 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/datanode/SimulatedFSDataset.java | 3 +-
.../server/datanode/TestSimulatedFSDataset.java | 47 ++++++++++++++++++++
3 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f39bd5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c8dde3..6bca0bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1684,6 +1684,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI.
(Archana T via wheat9)
+ HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the
+ correct namenode (Wei-Chiu Chuang via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f39bd5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 4de1be7..17ca6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -522,7 +523,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
private final Map<String, Map<Block, BInfo>> blockMap
- = new HashMap<String, Map<Block,BInfo>>();
+ = new ConcurrentHashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
private final SimulatedVolume volume;
private final String datanodeUuid;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f39bd5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 8dc80d5..dd7d239 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
import org.apache.hadoop.util.DataChecksum;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -338,4 +340,49 @@ public class TestSimulatedFSDataset {
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
+
+ @Test
+ public void testConcurrentAddBlockPool() throws InterruptedException,
+ IOException {
+ final String[] bpids = {"BP-TEST1-", "BP-TEST2-"};
+ final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
+ class AddBlockPoolThread extends Thread {
+ private int id;
+ private IOException ioe;
+ public AddBlockPoolThread(int id) {
+ super();
+ this.id = id;
+ }
+ public void test() throws InterruptedException, IOException {
+ this.join();
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+ public void run() {
+ for (int i=0; i < 10000; i++) {
+ // add different block pools concurrently
+ String newbpid = bpids[id] + i;
+ fsdataset.addBlockPool(newbpid, conf);
+ // and then add a block into the pool
+ ExtendedBlock block = new ExtendedBlock(newbpid,1);
+ try {
+ // it will throw an exception if the block pool is not found
+ fsdataset.createTemporary(StorageType.DEFAULT, block);
+ } catch (IOException ioe) {
+ // JUnit does not capture exception in non-main thread,
+ // so cache it and then let main thread throw later.
+ this.ioe = ioe;
+ }
+ assert(fsdataset.getReplicaString(newbpid,1) != "null");
+ }
+ }
+ };
+ AddBlockPoolThread t1 = new AddBlockPoolThread(0);
+ AddBlockPoolThread t2 = new AddBlockPoolThread(1);
+ t1.start();
+ t2.start();
+ t1.test();
+ t2.test();
+ }
}