You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/08/20 17:37:29 UTC
[hadoop] branch branch-3.1 updated: HDFS-14311. Multi-threading
conflict at layoutVersion when loading block pool storage. Contributed by
Yicong Cai.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 7a1f508 HDFS-14311. Multi-threading conflict at layoutVersion when loading block pool storage. Contributed by Yicong Cai.
7a1f508 is described below
commit 7a1f508f5822db093a273a07e1977745901059ba
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Tue Aug 20 10:08:55 2019 -0700
HDFS-14311. Multi-threading conflict at layoutVersion when loading block pool storage. Contributed by Yicong Cai.
(cherry picked from commit fbe87eddbc30fe5191c008b496fb83e51ef4ee4a)
(cherry picked from commit f78dcbaae5535e3e05c9f29986b8963ba262baa9)
---
.../hadoop/hdfs/server/datanode/DataStorage.java | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index a85ae32..2645e07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -445,22 +445,23 @@ public class DataStorage extends Storage {
StartupOption startOpt, ExecutorService executor) throws IOException {
final String bpid = nsInfo.getBlockPoolID();
final BlockPoolSliceStorage bpStorage = getBlockPoolSliceStorage(nsInfo);
+ Map<StorageLocation, List<Callable<StorageDirectory>>> upgradeCallableMap =
+ new HashMap<>();
final List<StorageDirectory> success = Lists.newArrayList();
final List<UpgradeTask> tasks = Lists.newArrayList();
for (StorageLocation dataDir : dataDirs) {
dataDir.makeBlockPoolDir(bpid, null);
try {
- final List<Callable<StorageDirectory>> callables = Lists.newArrayList();
+ final List<Callable<StorageDirectory>> sdCallables =
+ Lists.newArrayList();
final List<StorageDirectory> dirs = bpStorage.recoverTransitionRead(
- nsInfo, dataDir, startOpt, callables, datanode.getConf());
- if (callables.isEmpty()) {
+ nsInfo, dataDir, startOpt, sdCallables, datanode.getConf());
+ if (sdCallables.isEmpty()) {
for(StorageDirectory sd : dirs) {
success.add(sd);
}
} else {
- for(Callable<StorageDirectory> c : callables) {
- tasks.add(new UpgradeTask(dataDir, executor.submit(c)));
- }
+ upgradeCallableMap.put(dataDir, sdCallables);
}
} catch (IOException e) {
LOG.warn("Failed to add storage directory {} for block pool {}",
@@ -468,6 +469,13 @@ public class DataStorage extends Storage {
}
}
+ for (Map.Entry<StorageLocation, List<Callable<StorageDirectory>>> entry :
+ upgradeCallableMap.entrySet()) {
+ for(Callable<StorageDirectory> c : entry.getValue()) {
+ tasks.add(new UpgradeTask(entry.getKey(), executor.submit(c)));
+ }
+ }
+
if (!tasks.isEmpty()) {
LOG.info("loadBlockPoolSliceStorage: {} upgrade tasks", tasks.size());
for(UpgradeTask t : tasks) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org