You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/08/16 00:27:57 UTC
[03/50] [abbrv] hadoop git commit: HDFS-12157. Do fsyncDirectory(..)
outside of FSDataset lock. Contributed by inayakumar B.
HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by inayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69afa26f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69afa26f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69afa26f
Branch: refs/heads/YARN-6592
Commit: 69afa26f19adad4c630a307c274130eb8b697141
Parents: 1a18d5e
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Aug 9 09:03:51 2017 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Aug 9 09:03:51 2017 -0500
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 46 ++++++++++----------
1 file changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69afa26f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 53e2fc6..16df709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -991,8 +991,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
replicaInfo, smallBufferSize, conf);
// Finalize the copied files
- newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo,
- false);
+ newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
try (AutoCloseableLock lock = datasetLock.acquire()) {
// Increment numBlocks here as this block moved without knowing to BPS
FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume();
@@ -1295,7 +1294,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
replicaInfo.bumpReplicaGS(newGS);
// finalize the replica if RBW
if (replicaInfo.getState() == ReplicaState.RBW) {
- finalizeReplica(b.getBlockPoolId(), replicaInfo, false);
+ finalizeReplica(b.getBlockPoolId(), replicaInfo);
}
return replicaInfo;
}
@@ -1625,23 +1624,39 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
throws IOException {
+ ReplicaInfo replicaInfo = null;
+ ReplicaInfo finalizedReplicaInfo = null;
try (AutoCloseableLock lock = datasetLock.acquire()) {
if (Thread.interrupted()) {
// Don't allow data modifications from interrupted threads
throw new IOException("Cannot finalize block from Interrupted Thread");
}
- ReplicaInfo replicaInfo = getReplicaInfo(b);
+ replicaInfo = getReplicaInfo(b);
if (replicaInfo.getState() == ReplicaState.FINALIZED) {
// this is legal, when recovery happens on a file that has
// been opened for append but never modified
return;
}
- finalizeReplica(b.getBlockPoolId(), replicaInfo, fsyncDir);
+ finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo);
+ }
+ /*
+ * Sync the directory after rename from tmp/rbw to Finalized if
+ * configured. Though rename should be atomic operation, sync on both
+ * dest and src directories are done because IOUtils.fsync() calls
+ * directory's channel sync, not the journal itself.
+ */
+ if (fsyncDir && finalizedReplicaInfo instanceof FinalizedReplica
+ && replicaInfo instanceof LocalReplica) {
+ FinalizedReplica finalizedReplica =
+ (FinalizedReplica) finalizedReplicaInfo;
+ finalizedReplica.fsyncDirectory();
+ LocalReplica localReplica = (LocalReplica) replicaInfo;
+ localReplica.fsyncDirectory();
}
}
- private ReplicaInfo finalizeReplica(String bpid,
- ReplicaInfo replicaInfo, boolean fsyncDir) throws IOException {
+ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
+ throws IOException {
try (AutoCloseableLock lock = datasetLock.acquire()) {
ReplicaInfo newReplicaInfo = null;
if (replicaInfo.getState() == ReplicaState.RUR &&
@@ -1656,19 +1671,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
newReplicaInfo = v.addFinalizedBlock(
bpid, replicaInfo, replicaInfo, replicaInfo.getBytesReserved());
- /*
- * Sync the directory after rename from tmp/rbw to Finalized if
- * configured. Though rename should be atomic operation, sync on both
- * dest and src directories are done because IOUtils.fsync() calls
- * directory's channel sync, not the journal itself.
- */
- if (fsyncDir && newReplicaInfo instanceof FinalizedReplica
- && replicaInfo instanceof LocalReplica) {
- FinalizedReplica finalizedReplica = (FinalizedReplica) newReplicaInfo;
- finalizedReplica.fsyncDirectory();
- LocalReplica localReplica = (LocalReplica) replicaInfo;
- localReplica.fsyncDirectory();
- }
if (v.isTransientStorage()) {
releaseLockedMemory(
replicaInfo.getOriginalBytesReserved()
@@ -2634,11 +2636,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
newReplicaInfo.setNumBytes(newlength);
volumeMap.add(bpid, newReplicaInfo.getReplicaInfo());
- finalizeReplica(bpid, newReplicaInfo.getReplicaInfo(), false);
+ finalizeReplica(bpid, newReplicaInfo.getReplicaInfo());
}
}
// finalize the block
- return finalizeReplica(bpid, rur, false);
+ return finalizeReplica(bpid, rur);
}
@Override // FsDatasetSpi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org