You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/08/20 17:43:05 UTC
[hadoop] branch branch-2.9 updated: HDFS-14311. Multi-threading
conflict at layoutVersion when loading block pool storage. Contributed by
Yicong Cai.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new 0e7afe0 HDFS-14311. Multi-threading conflict at layoutVersion when loading block pool storage. Contributed by Yicong Cai.
0e7afe0 is described below
commit 0e7afe054ad67052359ab62757d6241d955b94d5
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Tue Aug 20 10:38:52 2019 -0700
HDFS-14311. Multi-threading conflict at layoutVersion when loading block pool storage. Contributed by Yicong Cai.
(cherry picked from commit bdab03cf711f75e88f80c21b3193fd5070cb6364)
---
.../hadoop/hdfs/server/datanode/DataStorage.java | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index c7cdf4b..a377589 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -447,6 +447,8 @@ public class DataStorage extends Storage {
StartupOption startOpt, ExecutorService executor) throws IOException {
final String bpid = nsInfo.getBlockPoolID();
final BlockPoolSliceStorage bpStorage = getBlockPoolSliceStorage(nsInfo);
+ Map<StorageLocation, List<Callable<StorageDirectory>>> upgradeCallableMap =
+ new HashMap<>();
final List<StorageDirectory> success = Lists.newArrayList();
final List<UpgradeTask> tasks = Lists.newArrayList();
for (StorageLocation dataDir : dataDirs) {
@@ -456,17 +458,16 @@ public class DataStorage extends Storage {
try {
makeBlockPoolDataDir(bpDataDirs, null);
- final List<Callable<StorageDirectory>> callables = Lists.newArrayList();
+ final List<Callable<StorageDirectory>> sdCallables =
+ Lists.newArrayList();
final List<StorageDirectory> dirs = bpStorage.recoverTransitionRead(
- nsInfo, bpDataDirs, startOpt, callables, datanode.getConf());
- if (callables.isEmpty()) {
+ nsInfo, bpDataDirs, startOpt, sdCallables, datanode.getConf());
+ if (sdCallables.isEmpty()) {
for(StorageDirectory sd : dirs) {
success.add(sd);
}
} else {
- for(Callable<StorageDirectory> c : callables) {
- tasks.add(new UpgradeTask(dataDir, executor.submit(c)));
- }
+ upgradeCallableMap.put(dataDir, sdCallables);
}
} catch (IOException e) {
LOG.warn("Failed to add storage directory " + dataDir
@@ -474,6 +475,13 @@ public class DataStorage extends Storage {
}
}
+ for (Map.Entry<StorageLocation, List<Callable<StorageDirectory>>> entry :
+ upgradeCallableMap.entrySet()) {
+ for(Callable<StorageDirectory> c : entry.getValue()) {
+ tasks.add(new UpgradeTask(entry.getKey(), executor.submit(c)));
+ }
+ }
+
if (!tasks.isEmpty()) {
LOG.info("loadBlockPoolSliceStorage: " + tasks.size() + " upgrade tasks");
for(UpgradeTask t : tasks) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org