You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2018/02/16 14:39:59 UTC
[08/50] [abbrv] hadoop git commit: HDFS-11695: [SPS]: Namenode failed
to start while loading SPS xAttrs from the edits log. Contributed by Surendra
Singh Lilhore.
HDFS-11695: [SPS]: Namenode failed to start while loading SPS xAttrs from the edits log. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7be73b1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7be73b1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7be73b1d
Branch: refs/heads/HDFS-10285
Commit: 7be73b1d8b076e0209079ccc78ad215bf25c4711
Parents: 7e1b0bf
Author: Uma Maheswara Rao G <um...@intel.com>
Authored: Mon May 22 21:39:43 2017 -0700
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Fri Feb 16 19:46:34 2018 +0530
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAttrOp.java | 91 ------------
.../namenode/FSDirSatisfyStoragePolicyOp.java | 145 +++++++++++++++++++
.../hdfs/server/namenode/FSDirXAttrOp.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 16 --
.../hdfs/server/namenode/FSNamesystem.java | 24 ++-
.../hadoop/hdfs/server/namenode/Namesystem.java | 10 ++
.../server/namenode/StoragePolicySatisfier.java | 4 +-
.../TestPersistentStoragePolicySatisfier.java | 90 +++++++++++-
.../namenode/TestStoragePolicySatisfier.java | 5 +-
9 files changed, 268 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 66d5f3d..201605f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -43,14 +42,12 @@ import com.google.common.collect.Lists;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
-import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY;
public class FSDirAttrOp {
static FileStatus setPermission(
@@ -200,29 +197,6 @@ public class FSDirAttrOp {
return fsd.getAuditFileInfo(iip);
}
- static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
- String src, boolean logRetryCache) throws IOException {
-
- FSPermissionChecker pc = fsd.getPermissionChecker();
- List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
- INodesInPath iip;
- fsd.writeLock();
- try {
-
- // check operation permission.
- iip = fsd.resolvePath(pc, src, DirOp.WRITE);
- if (fsd.isPermissionEnabled()) {
- fsd.checkPathAccess(pc, iip, FsAction.WRITE);
- }
- XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
- xAttrs.add(satisfyXAttr);
- } finally {
- fsd.writeUnlock();
- }
- fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
- return fsd.getAuditFileInfo(iip);
- }
-
static BlockStoragePolicy[] getStoragePolicies(BlockManager bm)
throws IOException {
return bm.getStoragePolicies();
@@ -487,71 +461,6 @@ public class FSDirAttrOp {
}
}
- static XAttr unprotectedSatisfyStoragePolicy(INodesInPath iip,
- BlockManager bm, FSDirectory fsd) throws IOException {
-
- final INode inode = FSDirectory.resolveLastINode(iip);
- final int snapshotId = iip.getLatestSnapshotId();
- final List<INode> candidateNodes = new ArrayList<>();
-
- // TODO: think about optimization here, label the dir instead
- // of the sub-files of the dir.
- if (inode.isFile()) {
- candidateNodes.add(inode);
- } else if (inode.isDirectory()) {
- for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
- if (node.isFile()) {
- candidateNodes.add(node);
- }
- }
- }
-
- // If node has satisfy xattr, then stop adding it
- // to satisfy movement queue.
- if (inodeHasSatisfyXAttr(candidateNodes)) {
- throw new IOException(
- "Cannot request to call satisfy storage policy on path "
- + iip.getPath()
- + ", as this file/dir was already called for satisfying "
- + "storage policy.");
- }
-
- final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
- final XAttr satisfyXAttr =
- XAttrHelper.buildXAttr(XATTR_SATISFY_STORAGE_POLICY);
- xattrs.add(satisfyXAttr);
-
- for (INode node : candidateNodes) {
- bm.satisfyStoragePolicy(node.getId());
- List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(node);
- List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(
- fsd, existingXAttrs, xattrs, EnumSet.of(XAttrSetFlag.CREATE));
- XAttrStorage.updateINodeXAttrs(node, newXAttrs, snapshotId);
- }
- return satisfyXAttr;
- }
-
- private static boolean inodeHasSatisfyXAttr(List<INode> candidateNodes) {
- // If the node is a directory and one of the child files
- // has satisfy xattr, then return true for this directory.
- for (INode inode : candidateNodes) {
- final XAttrFeature f = inode.getXAttrFeature();
- if (inode.isFile() &&
- f != null && f.getXAttr(XATTR_SATISFY_STORAGE_POLICY) != null) {
- return true;
- }
- }
- return false;
- }
-
- static void unprotectedRemoveSPSXAttr(INode inode, XAttr spsXAttr)
- throws IOException{
- List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
- existingXAttrs.remove(spsXAttr);
- XAttrStorage.updateINodeXAttrs(inode, existingXAttrs,
- INodesInPath.fromINode(inode).getLatestSnapshotId());
- }
-
private static void setDirStoragePolicy(
FSDirectory fsd, INodesInPath iip, byte policyId) throws IOException {
INode inode = FSDirectory.resolveLastINode(iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
new file mode 100644
index 0000000..81d337f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.XAttrHelper;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Helper class to perform storage policy satisfier related operations.
+ */
+final class FSDirSatisfyStoragePolicyOp {
+
+ /**
+ * Private constructor for preventing FSDirSatisfyStoragePolicyOp object
+ * creation. Static-only class.
+ */
+ private FSDirSatisfyStoragePolicyOp() {
+ }
+
+ static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
+ String src, boolean logRetryCache) throws IOException {
+
+ assert fsd.getFSNamesystem().hasWriteLock();
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+
+ // check operation permission.
+ iip = fsd.resolvePath(pc, src, DirOp.WRITE);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
+ xAttrs.add(satisfyXAttr);
+ fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static XAttr unprotectedSatisfyStoragePolicy(INodesInPath iip,
+ BlockManager bm, FSDirectory fsd) throws IOException {
+
+ final INode inode = FSDirectory.resolveLastINode(iip);
+ final int snapshotId = iip.getLatestSnapshotId();
+ final List<INode> candidateNodes = new ArrayList<>();
+
+ // TODO: think about optimization here, label the dir instead
+ // of the sub-files of the dir.
+ if (inode.isFile()) {
+ candidateNodes.add(inode);
+ } else if (inode.isDirectory()) {
+ for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
+ if (node.isFile()) {
+ candidateNodes.add(node);
+ }
+ }
+ }
+
+ // If node has satisfy xattr, then stop adding it
+ // to satisfy movement queue.
+ if (inodeHasSatisfyXAttr(candidateNodes)) {
+ throw new IOException(
+ "Cannot request to call satisfy storage policy on path "
+ + iip.getPath()
+ + ", as this file/dir was already called for satisfying "
+ + "storage policy.");
+ }
+
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ final XAttr satisfyXAttr = XAttrHelper
+ .buildXAttr(XATTR_SATISFY_STORAGE_POLICY);
+ xattrs.add(satisfyXAttr);
+
+ for (INode node : candidateNodes) {
+ bm.satisfyStoragePolicy(node.getId());
+ List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(node);
+ List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
+ xattrs, EnumSet.of(XAttrSetFlag.CREATE));
+ XAttrStorage.updateINodeXAttrs(node, newXAttrs, snapshotId);
+ }
+ return satisfyXAttr;
+ }
+
+ private static boolean inodeHasSatisfyXAttr(List<INode> candidateNodes) {
+ // If the node is a directory and one of the child files
+ // has satisfy xattr, then return true for this directory.
+ for (INode inode : candidateNodes) {
+ final XAttrFeature f = inode.getXAttrFeature();
+ if (inode.isFile() && f != null
+ && f.getXAttr(XATTR_SATISFY_STORAGE_POLICY) != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static void removeSPSXattr(FSDirectory fsd, INode inode, XAttr spsXAttr)
+ throws IOException {
+ try {
+ fsd.writeLock();
+ List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
+ existingXAttrs.remove(spsXAttr);
+ XAttrStorage.updateINodeXAttrs(inode, existingXAttrs, INodesInPath
+ .fromINode(inode).getLatestSnapshotId());
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(spsXAttr);
+ fsd.getEditLog().logRemoveXAttrs(inode.getFullPathName(), xAttrs, false);
+ } finally {
+ fsd.writeUnlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 6ddc07b..31afeea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -290,7 +290,7 @@ class FSDirXAttrOp {
// Add inode id to movement queue if xattrs contain satisfy xattr.
if (XATTR_SATISFY_STORAGE_POLICY.equals(xaName)) {
- FSDirAttrOp.unprotectedSatisfyStoragePolicy(iip,
+ FSDirSatisfyStoragePolicyOp.unprotectedSatisfyStoragePolicy(iip,
fsd.getBlockManager(), fsd);
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index f6f2001..c286ec1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -1423,22 +1423,6 @@ public class FSDirectory implements Closeable {
getBlockManager().satisfyStoragePolicy(inode.getId());
}
- /**
- * Remove the SPS xattr from the inode, retrieve the inode from the
- * block collection id.
- * @param id
- * - file block collection id.
- */
- public void removeSPSXattr(long id) throws IOException {
- final INode inode = getInode(id);
- final XAttrFeature xaf = inode.getXAttrFeature();
- final XAttr spsXAttr = xaf.getXAttr(XATTR_SATISFY_STORAGE_POLICY);
-
- if (spsXAttr != null) {
- FSDirAttrOp.unprotectedRemoveSPSXAttr(inode, spsXAttr);
- }
- }
-
private void addEncryptionZone(INodeWithAdditionalFields inode,
XAttrFeature xaf) {
if (xaf == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 09bf116..af43230 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2247,10 +2247,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ " by admin. Seek for an admin help to activate it "
+ "or use Mover tool.");
}
- FSDirAttrOp.satisfyStoragePolicy(dir, blockManager, src, logRetryCache);
+ FSDirSatisfyStoragePolicyOp.satisfyStoragePolicy(dir, blockManager, src,
+ logRetryCache);
} finally {
writeUnlock();
}
+ getEditLog().logSync();
}
/**
@@ -7754,6 +7756,26 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(true, operationName, src, null, auditStat);
}
+ @Override
+ public void removeXattr(long id, String xattrName) throws IOException {
+ writeLock();
+ try {
+ final INode inode = dir.getInode(id);
+ final XAttrFeature xaf = inode.getXAttrFeature();
+ if (xaf == null) {
+ return;
+ }
+ final XAttr spsXAttr = xaf.getXAttr(xattrName);
+
+ if (spsXAttr != null) {
+ FSDirSatisfyStoragePolicyOp.removeSPSXattr(dir, inode, spsXAttr);
+ }
+ } finally {
+ writeUnlock("removeXAttr");
+ }
+ getEditLog().logSync();
+ }
+
void checkAccess(String src, FsAction mode) throws IOException {
final String operationName = "checkAccess";
checkOperation(OperationCategory.READ);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index a2b07ca..e58fa72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -52,4 +54,12 @@ public interface Namesystem extends RwLock, SafeMode {
* @return true if valid write lease exists, otherwise return false.
*/
boolean isFileOpenedForWrite(String filePath);
+
+ /**
+ * Remove xAttr from the inode.
+ * @param id
+ * @param xattrName
+ * @throws IOException
+ */
+ void removeXattr(long id, String xattrName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 3b20314..9e2a4a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -829,6 +831,6 @@ public class StoragePolicySatisfier implements Runnable {
*/
public void postBlkStorageMovementCleanup(long trackId)
throws IOException {
- this.namesystem.getFSDirectory().removeSPSXattr(trackId);
+ this.namesystem.removeXattr(trackId, XATTR_SATISFY_STORAGE_POLICY);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
index 41c272c..bdf0159 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
@@ -32,6 +32,7 @@ import org.junit.Test;
import java.io.IOException;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY;
+import static org.junit.Assert.*;
/**
* Test persistence of satisfying files/directories.
@@ -60,11 +61,9 @@ public class TestPersistentStoragePolicySatisfier {
private static final String ALL_SSD = "ALL_SSD";
private static StorageType[][] storageTypes = new StorageType[][] {
- {StorageType.ARCHIVE, StorageType.DISK},
- {StorageType.DISK, StorageType.SSD},
- {StorageType.SSD, StorageType.RAM_DISK},
- {StorageType.ARCHIVE, StorageType.DISK},
- {StorageType.ARCHIVE, StorageType.SSD}
+ {StorageType.DISK, StorageType.ARCHIVE, StorageType.SSD},
+ {StorageType.DISK, StorageType.ARCHIVE, StorageType.SSD},
+ {StorageType.DISK, StorageType.ARCHIVE, StorageType.SSD}
};
private final int timeout = 300000;
@@ -94,10 +93,13 @@ public class TestPersistentStoragePolicySatisfier {
private void clusterSetUp(boolean isHAEnabled, Configuration newConf)
throws Exception {
conf = newConf;
+ conf.set(
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_KEY,
+ "3000");
final int dnNumber = storageTypes.length;
final short replication = 3;
MiniDFSCluster.Builder clusterBuilder = new MiniDFSCluster.Builder(conf)
- .storageTypes(storageTypes)
+ .storageTypes(storageTypes).storagesPerDatanode(3)
.numDataNodes(dnNumber);
if (isHAEnabled) {
clusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology());
@@ -277,9 +279,10 @@ public class TestPersistentStoragePolicySatisfier {
*/
@Test(timeout = 300000)
public void testWithFederationHA() throws Exception {
+ MiniDFSCluster haCluster = null;
try {
conf = new HdfsConfiguration();
- final MiniDFSCluster haCluster = new MiniDFSCluster
+ haCluster = new MiniDFSCluster
.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(2))
.storageTypes(storageTypes)
@@ -305,7 +308,14 @@ public class TestPersistentStoragePolicySatisfier {
testFileName, StorageType.ARCHIVE, 2, timeout, fs);
} finally {
- clusterShutdown();
+ if(fs != null) {
+ fs.close();
+ fs = null;
+ }
+ if(haCluster != null) {
+ haCluster.shutdown(true);
+ haCluster = null;
+ }
}
}
@@ -404,6 +414,70 @@ public class TestPersistentStoragePolicySatisfier {
}
/**
+ * Test loading of SPS xAttrs from the edits log when satisfyStoragePolicy
+ * called on child file and parent directory.
+ * 1. Create one directory and create one child file.
+ * 2. Set storage policy for child file and call
+ * satisfyStoragePolicy.
+ * 3. wait for SPS to remove xAttr for file child file.
+ * 4. Set storage policy for parent directory and call
+ * satisfyStoragePolicy.
+ * 5. restart the namenode.
+ * NameNode should be started successfully.
+ */
+ @Test(timeout = 300000)
+ public void testNameNodeRestartWhenSPSCalledOnChildFileAndParentDir()
+ throws Exception {
+ try {
+ clusterSetUp();
+ fs.setStoragePolicy(childFile, "COLD");
+ fs.satisfyStoragePolicy(childFile);
+ DFSTestUtil.waitExpectedStorageType(childFile.toUri().getPath(),
+ StorageType.ARCHIVE, 3, 30000, cluster.getFileSystem());
+ // wait for SPS to remove Xattr from file
+ Thread.sleep(30000);
+ fs.setStoragePolicy(childDir, "COLD");
+ fs.satisfyStoragePolicy(childDir);
+ try {
+ cluster.restartNameNodes();
+ } catch (Exception e) {
+ assertFalse(e.getMessage().contains(
+ "Cannot request to call satisfy storage policy"));
+ }
+ } finally {
+ clusterShutdown();
+ }
+ }
+
+ /**
+ * Test SPS when satisfyStoragePolicy called on child file and
+ * parent directory.
+ * 1. Create one parent directory and child directory.
+ * 2. Create some file in both the directory.
+ * 3. Set storage policy for parent directory and call
+ * satisfyStoragePolicy.
+ * 4. Set storage policy for child directory and call
+ * satisfyStoragePolicy.
+ * 5. restart the namenode.
+ * All the file blocks should satisfy the policy.
+ */
+ @Test(timeout = 300000)
+ public void testSPSOnChildAndParentDirectory() throws Exception {
+ try {
+ clusterSetUp();
+ fs.setStoragePolicy(parentDir, "COLD");
+ fs.satisfyStoragePolicy(childDir);
+ fs.satisfyStoragePolicy(parentDir);
+ DFSTestUtil.waitExpectedStorageType(childFileName, StorageType.ARCHIVE,
+ 3, 30000, cluster.getFileSystem());
+ DFSTestUtil.waitExpectedStorageType(parentFileName, StorageType.ARCHIVE,
+ 3, 30000, cluster.getFileSystem());
+ } finally {
+ clusterShutdown();
+ }
+ }
+
+ /**
* Restart the hole env and trigger the DataNode's heart beats.
* @throws Exception
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be73b1d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
index 8457e5b..fa954b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -855,7 +856,9 @@ public class TestStoragePolicySatisfier {
{StorageType.DISK, StorageType.SSD}};
int defaultStripedBlockSize =
- ErasureCodingPolicyManager.getSystemPolicies()[0].getCellSize() * 4;
+ StripedFileTestUtil.getDefaultECPolicy().getCellSize() * 4;
+ config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
+ StripedFileTestUtil.getDefaultECPolicy().getName());
config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultStripedBlockSize);
config.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
config.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org