You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/01/21 14:52:08 UTC
[1/4] hadoop git commit: HDFS-9625. set replication for empty file
failed when set storage policy (Contributed by DENG FEI)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 54df2a435 -> 845acfd96
refs/heads/branch-2.7 eb3dcc073 -> ca2993340
refs/heads/branch-2.8 de5175d21 -> 444967704
refs/heads/trunk 9eec6cbed -> b7372b716
HDFS-9625. set replication for empty file failed when set storage policy (Contributed by DENG FEI)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7372b71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7372b71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7372b71
Branch: refs/heads/trunk
Commit: b7372b7166a13111b98794602ca4c166dfd78d29
Parents: 9eec6cb
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 19:17:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 19:17:05 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++++
.../apache/hadoop/hdfs/TestSetrepIncreasing.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e6022fe..f1e6bdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2655,6 +2655,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
createRbw (ade via vinayakumarb)
+ HDFS-9625. set replication for empty file failed when set storage policy
+ (DENG FEI via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 6ca7a03..a00591f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -811,6 +811,10 @@ public class FSDirectory implements Closeable {
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
new EnumCounters<StorageType>(StorageType.class);
+ // empty file
+ if(dsDelta == 0){
+ return typeSpaceDeltas;
+ }
// Storage type and its quota are only available when storage policy is set
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index 0af1988..a055adc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -83,4 +83,23 @@ public class TestSetrepIncreasing {
public void testSetrepIncreasingSimulatedStorage() throws IOException {
setrep(3, 7, true);
}
+
+ @Test
+ public void testSetRepWithStoragePolicyOnEmptyFile() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ dfs.setStoragePolicy(d, "HOT");
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ dfs.setReplication(f, (short) 4);
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}
[4/4] hadoop git commit: HDFS-9625. set replication for empty file
failed when set storage policy (Contributed by DENG FEI)
Posted by vi...@apache.org.
HDFS-9625. set replication for empty file failed when set storage policy (Contributed by DENG FEI)
(cherry picked from commit b7372b7166a13111b98794602ca4c166dfd78d29)
(cherry picked from commit 845acfd96c8461ebb412cd1daec51e89ce9f1c18)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca299334
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca299334
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca299334
Branch: refs/heads/branch-2.7
Commit: ca299334052cec38e9b63c9711e53a9422c4d9f4
Parents: eb3dcc0
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 19:17:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 19:21:00 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++++
.../apache/hadoop/hdfs/TestSetrepIncreasing.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca299334/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 012080e..67e13a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -62,6 +62,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
createRbw (ade via vinayakumarb)
+ HDFS-9625. set replication for empty file failed when set storage policy
+ (DENG FEI via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca299334/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 0c4859f..58dc967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -741,6 +741,10 @@ public class FSDirectory implements Closeable {
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
new EnumCounters<StorageType>(StorageType.class);
+ // empty file
+ if(dsDelta == 0){
+ return typeSpaceDeltas;
+ }
// Storage type and its quota are only available when storage policy is set
if (storagePolicyID != BlockStoragePolicySuite.ID_UNSPECIFIED) {
BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca299334/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index 0af1988..a055adc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -83,4 +83,23 @@ public class TestSetrepIncreasing {
public void testSetrepIncreasingSimulatedStorage() throws IOException {
setrep(3, 7, true);
}
+
+ @Test
+ public void testSetRepWithStoragePolicyOnEmptyFile() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ dfs.setStoragePolicy(d, "HOT");
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ dfs.setReplication(f, (short) 4);
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}
[3/4] hadoop git commit: HDFS-9625. set replication for empty file
failed when set storage policy (Contributed by DENG FEI)
Posted by vi...@apache.org.
HDFS-9625. set replication for empty file failed when set storage policy (Contributed by DENG FEI)
(cherry picked from commit b7372b7166a13111b98794602ca4c166dfd78d29)
(cherry picked from commit 845acfd96c8461ebb412cd1daec51e89ce9f1c18)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44496770
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44496770
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44496770
Branch: refs/heads/branch-2.8
Commit: 4449677049b7aec0b1869d486cb57550ce3b2f66
Parents: de5175d
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 19:17:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 19:19:43 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++++
.../apache/hadoop/hdfs/TestSetrepIncreasing.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44496770/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 248783d..39dff21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1718,6 +1718,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
createRbw (ade via vinayakumarb)
+ HDFS-9625. set replication for empty file failed when set storage policy
+ (DENG FEI via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44496770/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index d885e2e..3e57a6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -811,6 +811,10 @@ public class FSDirectory implements Closeable {
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
new EnumCounters<StorageType>(StorageType.class);
+ // empty file
+ if(dsDelta == 0){
+ return typeSpaceDeltas;
+ }
// Storage type and its quota are only available when storage policy is set
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44496770/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index 0af1988..a055adc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -83,4 +83,23 @@ public class TestSetrepIncreasing {
public void testSetrepIncreasingSimulatedStorage() throws IOException {
setrep(3, 7, true);
}
+
+ @Test
+ public void testSetRepWithStoragePolicyOnEmptyFile() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ dfs.setStoragePolicy(d, "HOT");
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ dfs.setReplication(f, (short) 4);
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}
[2/4] hadoop git commit: HDFS-9625. set replication for empty file
failed when set storage policy (Contributed by DENG FEI)
Posted by vi...@apache.org.
HDFS-9625. set replication for empty file failed when set storage policy (Contributed by DENG FEI)
(cherry picked from commit b7372b7166a13111b98794602ca4c166dfd78d29)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/845acfd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/845acfd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/845acfd9
Branch: refs/heads/branch-2
Commit: 845acfd96c8461ebb412cd1daec51e89ce9f1c18
Parents: 54df2a4
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 19:17:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 19:18:36 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++++
.../apache/hadoop/hdfs/TestSetrepIncreasing.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/845acfd9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e734ba5..b3351e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1774,6 +1774,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
createRbw (ade via vinayakumarb)
+ HDFS-9625. set replication for empty file failed when set storage policy
+ (DENG FEI via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/845acfd9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 52af503..cdcec23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -811,6 +811,10 @@ public class FSDirectory implements Closeable {
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
new EnumCounters<StorageType>(StorageType.class);
+ // empty file
+ if(dsDelta == 0){
+ return typeSpaceDeltas;
+ }
// Storage type and its quota are only available when storage policy is set
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/845acfd9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index 0af1988..a055adc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -83,4 +83,23 @@ public class TestSetrepIncreasing {
public void testSetrepIncreasingSimulatedStorage() throws IOException {
setrep(3, 7, true);
}
+
+ @Test
+ public void testSetRepWithStoragePolicyOnEmptyFile() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ dfs.setStoragePolicy(d, "HOT");
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ dfs.setReplication(f, (short) 4);
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}