You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/02/22 20:47:56 UTC
[37/50] [abbrv] hadoop git commit: HDFS-9829. Erasure Coding: Improve
few exception handling logic of ErasureCodingWorker. Contributed by Rakesh R.
HDFS-9829. Erasure Coding: Improve few exception handling logic of ErasureCodingWorker. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6546d9e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6546d9e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6546d9e7
Branch: refs/heads/HDFS-1312
Commit: 6546d9e7ff73d2c81a803f2c61a1376a8c426987
Parents: ba1c9d4
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Feb 19 10:40:11 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Feb 19 10:40:28 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../erasurecode/ErasureCodingWorker.java | 37 ++++++++------------
2 files changed, 17 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6546d9e7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1f5d65d..a377243 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -225,6 +225,9 @@ Trunk (Unreleased)
HDFS-9795. OIV Delimited should show which files are ACL-enabled (lei)
+ HDFS-9829. Erasure Coding: Improve few exception handling logic of
+ ErasureCodingWorker. (Rakesh R via jing9)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6546d9e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 60c8417..b08aa2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -124,7 +124,7 @@ public final class ErasureCodingWorker {
}
private void initializeStripedReadThreadPool(int num) {
- LOG.debug("Using striped reads; pool threads=" + num);
+ LOG.debug("Using striped reads; pool threads={}", num);
EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num,
60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
@@ -150,7 +150,7 @@ public final class ErasureCodingWorker {
}
private void initializeStripedBlkReconstructionThreadPool(int num) {
- LOG.debug("Using striped block reconstruction; pool threads=" + num);
+ LOG.debug("Using striped block reconstruction; pool threads={}" + num);
EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL = new ThreadPoolExecutor(2, num,
60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
new Daemon.DaemonFactory() {
@@ -180,8 +180,8 @@ public final class ErasureCodingWorker {
EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL
.submit(new ReconstructAndTransferBlock(reconstructionInfo));
} catch (Throwable e) {
- LOG.warn("Failed to reconstruct striped block "
- + reconstructionInfo.getExtendedBlock().getLocalBlock(), e);
+ LOG.warn("Failed to reconstruct striped block {}",
+ reconstructionInfo.getExtendedBlock().getLocalBlock(), e);
}
}
}
@@ -476,12 +476,12 @@ public final class ErasureCodingWorker {
// Currently we don't check the acks for packets, this is similar as
// block replication.
} catch (Throwable e) {
- LOG.warn("Failed to reconstruct striped block: " + blockGroup, e);
+ LOG.warn("Failed to reconstruct striped block: {}", blockGroup, e);
} finally {
datanode.decrementXmitsInProgress();
// close block readers
for (StripedReader stripedReader : stripedReaders) {
- closeBlockReader(stripedReader.blockReader);
+ IOUtils.closeStream(stripedReader.blockReader);
}
for (int i = 0; i < targets.length; i++) {
IOUtils.closeStream(targetOutputStreams[i]);
@@ -588,7 +588,7 @@ public final class ErasureCodingWorker {
// If read failed for some source DN, we should not use it anymore
// and schedule read from another source DN.
StripedReader failedReader = stripedReaders.get(result.index);
- closeBlockReader(failedReader.blockReader);
+ IOUtils.closeStream(failedReader.blockReader);
failedReader.blockReader = null;
resultIndex = scheduleNewRead(used, reconstructLength,
corruptionMap);
@@ -609,6 +609,8 @@ public final class ErasureCodingWorker {
}
} catch (InterruptedException e) {
LOG.info("Read data interrupted.", e);
+ cancelReads(futures.keySet());
+ futures.clear();
break;
}
}
@@ -736,7 +738,7 @@ public final class ErasureCodingWorker {
StripedReader r = stripedReaders.get(i);
toRead = getReadLength(liveIndices[i], reconstructLen);
if (toRead > 0) {
- closeBlockReader(r.blockReader);
+ IOUtils.closeStream(r.blockReader);
r.blockReader = newBlockReader(
getBlock(blockGroup, liveIndices[i]), positionInBlock,
sources[i]);
@@ -784,8 +786,8 @@ public final class ErasureCodingWorker {
actualReadFromBlock(reader.blockReader, buf);
return null;
} catch (ChecksumException e) {
- LOG.warn("Found Checksum error for " + reader.block + " from "
- + reader.source + " at " + e.getPos());
+ LOG.warn("Found Checksum error for {} from {} at {}", reader.block,
+ reader.source, e.getPos());
addCorruptedBlock(reader.block, reader.source, corruptionMap);
throw e;
} catch (IOException e) {
@@ -837,17 +839,6 @@ public final class ErasureCodingWorker {
}
}
- // close block reader
- private void closeBlockReader(BlockReader blockReader) {
- try {
- if (blockReader != null) {
- blockReader.close();
- }
- } catch (IOException e) {
- // ignore
- }
- }
-
private InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) {
return NetUtils.createSocketAddr(dnInfo.getXferAddr(
datanode.getDnConf().getConnectToDnViaHostname()));
@@ -867,8 +858,6 @@ public final class ErasureCodingWorker {
* read directly from DN and need to check the replica is FINALIZED
* state, notice we should not use short-circuit local read which
* requires config for domain-socket in UNIX or legacy config in Windows.
- *
- * TODO: add proper tracer
*/
return RemoteBlockReader2.newBlockReader(
"dummy", block, blockToken, offsetInBlock,
@@ -876,6 +865,8 @@ public final class ErasureCodingWorker {
"", newConnectedPeer(block, dnAddr, blockToken, dnInfo), dnInfo,
null, cachingStrategy, datanode.getTracer());
} catch (IOException e) {
+ LOG.debug("Exception while creating remote block reader, datanode {}",
+ dnInfo, e);
return null;
}
}