You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/05/26 21:35:12 UTC
[42/50] [abbrv] hadoop git commit: HDFS-8441. Erasure Coding: make
condition check earlier for setReplication. (waltersu4549)
HDFS-8441. Erasure Coding: make condition check earlier for setReplication. (waltersu4549)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d734df2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d734df2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d734df2
Branch: refs/heads/HDFS-7285
Commit: 3d734df24cba53ec56074b4d28e3bcdce7d2894e
Parents: 7af05a3
Author: Walter Su <wa...@apache.org>
Authored: Fri May 22 10:25:53 2015 +0800
Committer: Zhe Zhang <zh...@cloudera.com>
Committed: Tue May 26 12:07:09 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../hdfs/server/namenode/FSDirAttrOp.java | 7 +++++-
.../hdfs/server/namenode/FSNamesystem.java | 20 +++++++++++++++-
.../hadoop/hdfs/server/namenode/INodeFile.java | 2 --
.../hadoop/hdfs/TestErasureCodingZones.java | 24 +++++++++++++++++++-
5 files changed, 51 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d734df2/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index c986f19..d71b9c3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -250,3 +250,6 @@
HDFS-8294. Erasure Coding: Fix Findbug warnings present in erasure coding.
(Rakesh R via zhz)
+
+ HDFS-8441. Erasure Coding: make condition check earlier for setReplication.
+ (waltersu4549)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d734df2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 9abb9fa..d34a0fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -380,7 +380,7 @@ public class FSDirAttrOp {
static BlockInfoContiguous[] unprotectedSetReplication(
FSDirectory fsd, String src, short replication, short[] blockRepls)
throws QuotaExceededException, UnresolvedLinkException,
- SnapshotAccessControlException {
+ SnapshotAccessControlException, UnsupportedActionException {
assert fsd.hasWriteLock();
final INodesInPath iip = fsd.getINodesInPath4Write(src, true);
@@ -389,6 +389,11 @@ public class FSDirAttrOp {
return null;
}
INodeFile file = inode.asFile();
+ if (file.isStriped()) {
+ throw new UnsupportedActionException(
+ "Cannot set replication to a file with striped blocks");
+ }
+
final short oldBR = file.getPreferredBlockReplication();
// before setFileReplication, check for increasing block replication.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d734df2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e9bb2f7..a53cd61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2404,7 +2404,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException(src);
}
- blockManager.verifyReplication(src, replication, clientMachine);
+
+ checkOperation(OperationCategory.READ);
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ if (!isInECZone(src)) {
+ blockManager.verifyReplication(src, replication, clientMachine);
+ }
+ } finally {
+ readUnlock();
+ }
+
checkOperation(OperationCategory.WRITE);
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
@@ -7605,6 +7616,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(success, "createErasureCodingZone", srcArg, null, resultingStat);
}
+ private boolean isInECZone(String src) throws IOException {
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = FSDirectory.resolvePath(src, pathComponents, dir);
+ final INodesInPath iip = dir.getINodesInPath(src, true);
+ return dir.isInECZone(iip);
+ }
+
/**
* Get the erasure coding information for specified src
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d734df2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 41287e8..4688001 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -429,8 +429,6 @@ public class INodeFile extends INodeWithAdditionalFields
/** Set the replication factor of this file. */
public final INodeFile setFileReplication(short replication,
int latestSnapshotId) throws QuotaExceededException {
- Preconditions.checkState(!isStriped(),
- "Cannot set replication to a file with striped blocks");
recordModification(latestSnapshotId);
setFileReplication(replication);
return this;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d734df2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
index a7c3cd4..480791e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.ErasureCodingInfo;
@@ -35,6 +36,7 @@ import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.*;
public class TestErasureCodingZones {
+ private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
@@ -42,7 +44,7 @@ public class TestErasureCodingZones {
@Before
public void setupCluster() throws IOException {
- Configuration conf = new HdfsConfiguration();
+ conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
cluster = new MiniDFSCluster.Builder(conf).
numDataNodes(1).build();
@@ -150,6 +152,26 @@ public class TestErasureCodingZones {
}
@Test
+ public void testReplication() throws IOException {
+ final Path testDir = new Path("/ec");
+ fs.mkdir(testDir, FsPermission.getDirDefault());
+ fs.createErasureCodingZone(testDir, null, 0);
+ final Path fooFile = new Path(testDir, "foo");
+ // create ec file with replication=0
+ fs.create(fooFile, FsPermission.getFileDefault(), true,
+ conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ (short)0, fs.getDefaultBlockSize(fooFile), null);
+
+ try {
+ fs.setReplication(fooFile, (short) 3);
+ fail("Shouldn't allow to set replication to a file with striped blocks");
+ } catch (IOException e) {
+ assertExceptionContains(
+ "Cannot set replication to a file with striped blocks", e);
+ }
+ }
+
+ @Test
public void testGetErasureCodingInfoWithSystemDefaultSchema() throws Exception {
String src = "/ec";
final Path ecDir = new Path(src);