You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/19 19:03:44 UTC
[1/4] git commit: HDFS-6948. DN rejects blocks if it has older UC
block. Contributed by Eric Payne.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 e4d29fda7 -> 222bf0fe6
HDFS-6948. DN rejects blocks if it has older UC block. Contributed by
Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f02d934f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f02d934f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f02d934f
Branch: refs/heads/HDFS-6581
Commit: f02d934fedf00f0ce43d6f3f9b06d89ccc6851a5
Parents: 6fe5c6b
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 08:50:43 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 08:50:43 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++++++++---
.../fsdataset/impl/TestWriteToReplica.java | 25 +++++++++++++++++++-
3 files changed, 38 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 436d2f0..9d76c37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -564,6 +564,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6970. Move startFile EDEK retries to the DFSClient. (wang)
+ HDFS-6948. DN rejects blocks if it has older UC block
+ (Eric Payne via kihwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d28d616..a2179dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1090,9 +1090,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ExtendedBlock b) throws IOException {
ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId());
if (replicaInfo != null) {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " already exists in state " + replicaInfo.getState() +
- " and thus cannot be created.");
+ if (replicaInfo.getGenerationStamp() < b.getGenerationStamp()
+ && replicaInfo instanceof ReplicaInPipeline) {
+ // Stop the previous writer
+ ((ReplicaInPipeline)replicaInfo)
+ .stopWriter(datanode.getDnConf().getXceiverStopTimeout());
+ invalidate(b.getBlockPoolId(), new Block[]{replicaInfo});
+ } else {
+ throw new ReplicaAlreadyExistsException("Block " + b +
+ " already exists in state " + replicaInfo.getState() +
+ " and thus cannot be created.");
+ }
}
FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02d934f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index e6a03d2..a870aa9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -111,7 +111,7 @@ public class TestWriteToReplica {
// test writeToTemporary
@Test
- public void testWriteToTempoary() throws Exception {
+ public void testWriteToTemporary() throws Exception {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).build();
try {
cluster.waitActive();
@@ -475,5 +475,28 @@ public class TestWriteToReplica {
}
dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+
+ try {
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.fail("Should not have created a replica that had already been "
+ + "created " + blocks[NON_EXISTENT]);
+ } catch (Exception e) {
+ Assert.assertTrue(
+ e.getMessage().contains(blocks[NON_EXISTENT].getBlockName()));
+ Assert.assertTrue(e instanceof ReplicaAlreadyExistsException);
+ }
+
+ long newGenStamp = blocks[NON_EXISTENT].getGenerationStamp() * 10;
+ blocks[NON_EXISTENT].setGenerationStamp(newGenStamp);
+ try {
+ ReplicaInPipeline replicaInfo =
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp);
+ Assert.assertTrue(
+ replicaInfo.getBlockId() == blocks[NON_EXISTENT].getBlockId());
+ } catch (ReplicaAlreadyExistsException e) {
+ Assert.fail("createRbw() Should have removed the block with the older "
+ + "genstamp and replaced it with the newer one: " + blocks[NON_EXISTENT]);
+ }
}
}
[3/4] git commit: HDFS-7096. Fix TestRpcProgramNfs3 to use
DFS_ENCRYPTION_KEY_PROVIDER_URI (clamb via cmccabe)
Posted by ar...@apache.org.
HDFS-7096. Fix TestRpcProgramNfs3 to use DFS_ENCRYPTION_KEY_PROVIDER_URI (clamb via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25fd69a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25fd69a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25fd69a8
Branch: refs/heads/HDFS-6581
Commit: 25fd69a8f9defb21a5c0e3b7243657d9e8080319
Parents: bf27b9c
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Sep 19 09:16:05 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Fri Sep 19 09:20:25 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 3 ++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25fd69a8/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
index 68efac2..acd47fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -131,7 +132,7 @@ public class TestRpcProgramNfs3 {
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- config.set(KeyProviderFactory.KEY_PROVIDER_PATH,
+ config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25fd69a8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ba1a272..9d4d8ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -779,6 +779,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7065. Pipeline close recovery race can cause block corruption (kihwal)
+ HDFS-7096. Fix TestRpcProgramNfs3 to use DFS_ENCRYPTION_KEY_PROVIDER_URI
+ (clamb via cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
[2/4] git commit: HDFS-7065. Pipeline close recovery race can cause
block corruption. Contributed by Kihwal Lee.
Posted by ar...@apache.org.
HDFS-7065. Pipeline close recovery race can cause block corruption.
Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf27b9ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf27b9ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf27b9ca
Branch: refs/heads/HDFS-6581
Commit: bf27b9ca574592ef603e126bacb9b6a37c9eb5c6
Parents: f02d934
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 09:00:13 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 09:00:13 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf27b9ca/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9d76c37..ba1a272 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -777,6 +777,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6840. Clients are always sent to the same datanode when read
is off rack. (wang)
+ HDFS-7065. Pipeline close recovery race can cause block corruption (kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf27b9ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index a2179dc..6e81082 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -904,7 +904,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public String recoverClose(ExtendedBlock b, long newGS,
+ public synchronized String recoverClose(ExtendedBlock b, long newGS,
long expectedBlockLen) throws IOException {
LOG.info("Recover failed close " + b);
// check replica's state
[4/4] git commit: Merge branch 'trunk' into HDFS-6581
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-6581
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/222bf0fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/222bf0fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/222bf0fe
Branch: refs/heads/HDFS-6581
Commit: 222bf0fe6706ee43964fd39b8315c1a339fbc84a
Parents: e4d29fd 25fd69a
Author: arp <ar...@apache.org>
Authored: Fri Sep 19 10:02:27 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Fri Sep 19 10:02:27 2014 -0700
----------------------------------------------------------------------
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 3 ++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 +++++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 16 +++++++++----
.../fsdataset/impl/TestWriteToReplica.java | 25 +++++++++++++++++++-
4 files changed, 46 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/222bf0fe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/222bf0fe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------