You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/12/08 10:15:48 UTC
[25/29] hadoop git commit: HDFS-7459. Consolidate cache-related
implementation in FSNamesystem into a single class. Contributed by Haohui
Mai.
HDFS-7459. Consolidate cache-related implementation in FSNamesystem into a single class. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e227fb8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e227fb8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e227fb8f
Branch: refs/heads/HDFS-EC
Commit: e227fb8fbcd414717faded9454b8ef813f7aafea
Parents: 0707e4e
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Dec 5 18:35:45 2014 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Dec 5 18:37:07 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSNDNCacheOp.java | 124 ++++++++++++++++
.../hdfs/server/namenode/FSNamesystem.java | 140 ++++++-------------
3 files changed, 173 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e227fb8f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d4db732..87b02c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -435,6 +435,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7474. Avoid resolving path in FSPermissionChecker. (jing9)
+ HDFS-7459. Consolidate cache-related implementation in FSNamesystem into
+ a single class. (wheat9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e227fb8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNDNCacheOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNDNCacheOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNDNCacheOp.java
new file mode 100644
index 0000000..093ee74
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNDNCacheOp.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+class FSNDNCacheOp {
+ static CacheDirectiveInfo addCacheDirective(
+ FSNamesystem fsn, CacheManager cacheManager,
+ CacheDirectiveInfo directive, EnumSet<CacheFlag> flags,
+ boolean logRetryCache)
+ throws IOException {
+
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+
+ if (directive.getId() != null) {
+ throw new IOException("addDirective: you cannot specify an ID " +
+ "for this operation.");
+ }
+ CacheDirectiveInfo effectiveDirective =
+ cacheManager.addDirective(directive, pc, flags);
+ fsn.getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
+ logRetryCache);
+ return effectiveDirective;
+ }
+
+ static void modifyCacheDirective(
+ FSNamesystem fsn, CacheManager cacheManager, CacheDirectiveInfo directive,
+ EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+
+ cacheManager.modifyDirective(directive, pc, flags);
+ fsn.getEditLog().logModifyCacheDirectiveInfo(directive, logRetryCache);
+ }
+
+ static void removeCacheDirective(
+ FSNamesystem fsn, CacheManager cacheManager, long id,
+ boolean logRetryCache)
+ throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+
+ cacheManager.removeDirective(id, pc);
+ fsn.getEditLog().logRemoveCacheDirectiveInfo(id, logRetryCache);
+ }
+
+ static BatchedListEntries<CacheDirectiveEntry> listCacheDirectives(
+ FSNamesystem fsn, CacheManager cacheManager,
+ long startId, CacheDirectiveInfo filter) throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+ return cacheManager.listCacheDirectives(startId, filter, pc);
+ }
+
+ static CachePoolInfo addCachePool(
+ FSNamesystem fsn, CacheManager cacheManager, CachePoolInfo req,
+ boolean logRetryCache)
+ throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+
+ if (pc != null) {
+ pc.checkSuperuserPrivilege();
+ }
+ CachePoolInfo info = cacheManager.addCachePool(req);
+ fsn.getEditLog().logAddCachePool(info, logRetryCache);
+ return info;
+ }
+
+ static void modifyCachePool(
+ FSNamesystem fsn, CacheManager cacheManager, CachePoolInfo req,
+ boolean logRetryCache) throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+ if (pc != null) {
+ pc.checkSuperuserPrivilege();
+ }
+ cacheManager.modifyCachePool(req);
+ fsn.getEditLog().logModifyCachePool(req, logRetryCache);
+ }
+
+ static void removeCachePool(
+ FSNamesystem fsn, CacheManager cacheManager, String cachePoolName,
+ boolean logRetryCache) throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+ if (pc != null) {
+ pc.checkSuperuserPrivilege();
+ }
+ cacheManager.removeCachePool(cachePoolName);
+ fsn.getEditLog().logRemoveCachePool(cachePoolName, logRetryCache);
+ }
+
+ static BatchedListEntries<CachePoolEntry> listCachePools(
+ FSNamesystem fsn, CacheManager cacheManager, String prevKey)
+ throws IOException {
+ final FSPermissionChecker pc = getFsPermissionChecker(fsn);
+ return cacheManager.listCachePools(pc, prevKey);
+ }
+
+ private static FSPermissionChecker getFsPermissionChecker(FSNamesystem fsn)
+ throws AccessControlException {
+ return fsn.isPermissionEnabled() ? fsn.getPermissionChecker() : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e227fb8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0a558dd..982798f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1354,7 +1354,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
}
-
+
+ boolean isPermissionEnabled() {
+ return isPermissionEnabled;
+ }
+
/**
* We already know that the safemode is on. We will throw a RetriableException
* if the safemode is not manual or caused by low resource.
@@ -3607,7 +3611,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return status;
}
- private FSPermissionChecker getPermissionChecker()
+ FSPermissionChecker getPermissionChecker()
throws AccessControlException {
return dir.getPermissionChecker();
}
@@ -7541,52 +7545,38 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
EnumSet<CacheFlag> flags, boolean logRetryCache)
throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc = isPermissionEnabled ?
- getPermissionChecker() : null;
-
+ CacheDirectiveInfo effectiveDirective = null;
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
}
- boolean success = false;
writeLock();
- String effectiveDirectiveStr = null;
- Long result = null;
try {
checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException(
"Cannot add cache directive", safeMode);
}
- if (directive.getId() != null) {
- throw new IOException("addDirective: you cannot specify an ID " +
- "for this operation.");
- }
- CacheDirectiveInfo effectiveDirective =
- cacheManager.addDirective(directive, pc, flags);
- getEditLog().logAddCacheDirectiveInfo(effectiveDirective, logRetryCache);
- result = effectiveDirective.getId();
- effectiveDirectiveStr = effectiveDirective.toString();
- success = true;
+ effectiveDirective = FSNDNCacheOp.addCacheDirective(this, cacheManager,
+ directive, flags, logRetryCache);
} finally {
writeUnlock();
+ boolean success = effectiveDirective != null;
if (success) {
getEditLog().logSync();
}
- if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCacheDirective", effectiveDirectiveStr, null, null);
- }
+ String effectiveDirectiveStr = effectiveDirective != null ?
+ effectiveDirective.toString() : null;
+ logAuditEvent(success, "addCacheDirective", effectiveDirectiveStr,
+ null, null);
}
- return result;
+ return effectiveDirective != null ? effectiveDirective.getId() : 0;
}
void modifyCacheDirective(CacheDirectiveInfo directive,
EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc = isPermissionEnabled ?
- getPermissionChecker() : null;
boolean success = false;
-
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
}
@@ -7597,26 +7587,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new SafeModeException(
"Cannot add cache directive", safeMode);
}
- cacheManager.modifyDirective(directive, pc, flags);
- getEditLog().logModifyCacheDirectiveInfo(directive, logRetryCache);
+ FSNDNCacheOp.modifyCacheDirective(this, cacheManager, directive, flags,
+ logRetryCache);
success = true;
} finally {
writeUnlock();
if (success) {
getEditLog().logSync();
}
- if (isAuditEnabled() && isExternalInvocation()) {
- String idStr = "{id: " + directive.getId().toString() + "}";
- logAuditEvent(success, "modifyCacheDirective", idStr, directive.toString(), null);
- }
+ String idStr = "{id: " + directive.getId().toString() + "}";
+ logAuditEvent(success, "modifyCacheDirective", idStr,
+ directive.toString(), null);
}
}
void removeCacheDirective(long id, boolean logRetryCache) throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc = isPermissionEnabled ?
- getPermissionChecker() : null;
-
boolean success = false;
writeLock();
try {
@@ -7625,16 +7611,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new SafeModeException(
"Cannot remove cache directives", safeMode);
}
- cacheManager.removeDirective(id, pc);
- getEditLog().logRemoveCacheDirectiveInfo(id, logRetryCache);
+ FSNDNCacheOp.removeCacheDirective(this, cacheManager, id, logRetryCache);
success = true;
} finally {
writeUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- String idStr = "{id: " + Long.toString(id) + "}";
- logAuditEvent(success, "removeCacheDirective", idStr, null,
- null);
- }
+ String idStr = "{id: " + Long.toString(id) + "}";
+ logAuditEvent(success, "removeCacheDirective", idStr, null,
+ null);
}
getEditLog().logSync();
}
@@ -7642,33 +7625,26 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
BatchedListEntries<CacheDirectiveEntry> listCacheDirectives(
long startId, CacheDirectiveInfo filter) throws IOException {
checkOperation(OperationCategory.READ);
- final FSPermissionChecker pc = isPermissionEnabled ?
- getPermissionChecker() : null;
BatchedListEntries<CacheDirectiveEntry> results;
cacheManager.waitForRescanIfNeeded();
readLock();
boolean success = false;
try {
checkOperation(OperationCategory.READ);
- results =
- cacheManager.listCacheDirectives(startId, filter, pc);
+ results = FSNDNCacheOp.listCacheDirectives(this, cacheManager, startId,
+ filter);
success = true;
} finally {
readUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "listCacheDirectives", filter.toString(), null,
- null);
- }
+ logAuditEvent(success, "listCacheDirectives", filter.toString(), null,
+ null);
}
return results;
}
- public void addCachePool(CachePoolInfo req, boolean logRetryCache)
+ void addCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc = isPermissionEnabled ?
- getPermissionChecker() : null;
-
writeLock();
boolean success = false;
String poolInfoStr = null;
@@ -7678,29 +7654,21 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new SafeModeException(
"Cannot add cache pool " + req.getPoolName(), safeMode);
}
- if (pc != null) {
- pc.checkSuperuserPrivilege();
- }
- CachePoolInfo info = cacheManager.addCachePool(req);
+ CachePoolInfo info = FSNDNCacheOp.addCachePool(this, cacheManager, req,
+ logRetryCache);
poolInfoStr = info.toString();
- getEditLog().logAddCachePool(info, logRetryCache);
success = true;
} finally {
writeUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "addCachePool", poolInfoStr, null, null);
- }
+ logAuditEvent(success, "addCachePool", poolInfoStr, null, null);
}
getEditLog().logSync();
}
- public void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
+ void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc =
- isPermissionEnabled ? getPermissionChecker() : null;
-
writeLock();
boolean success = false;
try {
@@ -7709,29 +7677,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new SafeModeException(
"Cannot modify cache pool " + req.getPoolName(), safeMode);
}
- if (pc != null) {
- pc.checkSuperuserPrivilege();
- }
- cacheManager.modifyCachePool(req);
- getEditLog().logModifyCachePool(req, logRetryCache);
+ FSNDNCacheOp.modifyCachePool(this, cacheManager, req, logRetryCache);
success = true;
} finally {
writeUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- String poolNameStr = "{poolName: " + req.getPoolName() + "}";
- logAuditEvent(success, "modifyCachePool", poolNameStr, req.toString(), null);
- }
+ String poolNameStr = "{poolName: " +
+ (req == null ? null : req.getPoolName()) + "}";
+ logAuditEvent(success, "modifyCachePool", poolNameStr,
+ req == null ? null : req.toString(), null);
}
getEditLog().logSync();
}
- public void removeCachePool(String cachePoolName, boolean logRetryCache)
+ void removeCachePool(String cachePoolName, boolean logRetryCache)
throws IOException {
checkOperation(OperationCategory.WRITE);
- final FSPermissionChecker pc =
- isPermissionEnabled ? getPermissionChecker() : null;
-
writeLock();
boolean success = false;
try {
@@ -7740,27 +7701,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new SafeModeException(
"Cannot remove cache pool " + cachePoolName, safeMode);
}
- if (pc != null) {
- pc.checkSuperuserPrivilege();
- }
- cacheManager.removeCachePool(cachePoolName);
- getEditLog().logRemoveCachePool(cachePoolName, logRetryCache);
+ FSNDNCacheOp.removeCachePool(this, cacheManager, cachePoolName,
+ logRetryCache);
success = true;
} finally {
writeUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- String poolNameStr = "{poolName: " + cachePoolName + "}";
- logAuditEvent(success, "removeCachePool", poolNameStr, null, null);
- }
+ String poolNameStr = "{poolName: " + cachePoolName + "}";
+ logAuditEvent(success, "removeCachePool", poolNameStr, null, null);
}
getEditLog().logSync();
}
- public BatchedListEntries<CachePoolEntry> listCachePools(String prevKey)
+ BatchedListEntries<CachePoolEntry> listCachePools(String prevKey)
throws IOException {
- final FSPermissionChecker pc =
- isPermissionEnabled ? getPermissionChecker() : null;
BatchedListEntries<CachePoolEntry> results;
checkOperation(OperationCategory.READ);
boolean success = false;
@@ -7768,13 +7722,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
readLock();
try {
checkOperation(OperationCategory.READ);
- results = cacheManager.listCachePools(pc, prevKey);
+ results = FSNDNCacheOp.listCachePools(this, cacheManager, prevKey);
success = true;
} finally {
readUnlock();
- if (isAuditEnabled() && isExternalInvocation()) {
- logAuditEvent(success, "listCachePools", null, null, null);
- }
+ logAuditEvent(success, "listCachePools", null, null, null);
}
return results;
}