You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/05/29 00:26:39 UTC
hadoop git commit: Fixed more FilesSystemRMStateStore issues.
Contributed by Vinod Kumar Vavilapalli.
Repository: hadoop
Updated Branches:
refs/heads/trunk 860b8373c -> 9acd24fec
Fixed more FilesSystemRMStateStore issues. Contributed by Vinod Kumar Vavilapalli.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9acd24fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9acd24fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9acd24fe
Branch: refs/heads/trunk
Commit: 9acd24fec408c182683968ae2c75ca2c38866a60
Parents: 860b837
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu May 28 15:25:12 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu May 28 15:25:56 2015 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +++++++++++
.../recovery/FileSystemRMStateStore.java | 20 +++++++++----------
.../recovery/TestFSRMStateStore.java | 21 ++++++++++----------
3 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9acd24fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 17cb020..0268c26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -268,6 +268,18 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /**
+ * Enable/disable intermediate-data encryption at YARN level. For now, this
+ * only is used by the FileSystemRMStateStore to setup right file-system
+ * security attributes.
+ */
+ @Private
+ public static final String YARN_INTERMEDIATE_DATA_ENCRYPTION = YARN_PREFIX
+ + "intermediate-data-encryption.enable";
+
+ @Private
+ public static final Boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION = false;
+
/** The address of the RM admin interface.*/
public static final String RM_ADMIN_ADDRESS =
RM_PREFIX + "admin.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9acd24fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 0d97d6f..b972791 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
+
import com.google.common.annotations.VisibleForTesting;
@Private
@@ -100,7 +101,8 @@ public class FileSystemRMStateStore extends RMStateStore {
private Path dtSequenceNumberPath = null;
private int fsNumRetries;
private long fsRetryInterval;
- private volatile boolean isHDFS;
+ private boolean intermediateEncryptionEnabled =
+ YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION;
@VisibleForTesting
Path fsWorkingPath;
@@ -121,6 +123,9 @@ public class FileSystemRMStateStore extends RMStateStore {
fsRetryInterval =
conf.getLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS);
+ intermediateEncryptionEnabled =
+ conf.getBoolean(YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION,
+ YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION);
}
@Override
@@ -145,17 +150,11 @@ public class FileSystemRMStateStore extends RMStateStore {
}
fs = fsWorkingPath.getFileSystem(fsConf);
- isHDFS = fs.getScheme().toLowerCase().contains("hdfs");
mkdirsWithRetries(rmDTSecretManagerRoot);
mkdirsWithRetries(rmAppRoot);
mkdirsWithRetries(amrmTokenSecretManagerRoot);
}
- @VisibleForTesting
- void setIsHDFS(boolean isHDFS) {
- this.isHDFS = isHDFS;
- }
-
@Override
protected synchronized void closeInternal() throws Exception {
closeWithRetries();
@@ -856,10 +855,11 @@ public class FileSystemRMStateStore extends RMStateStore {
private void setUnreadableBySuperuserXattrib(Path p)
throws IOException {
- if (isHDFS &&
- !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
+ if (fs.getScheme().toLowerCase().contains("hdfs")
+ && intermediateEncryptionEnabled
+ && !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
fs.setXAttr(p, UNREADABLE_BY_SUPERUSER_XATTRIB, null,
- EnumSet.of(XAttrSetFlag.CREATE));
+ EnumSet.of(XAttrSetFlag.CREATE));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9acd24fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 2dba8b4..c842294 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -26,11 +26,6 @@ import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,17 +33,22 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Assert;
import org.junit.Test;
public class TestFSRMStateStore extends RMStateStoreTestBase {
@@ -111,6 +111,10 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
900L);
+ if (adminCheckEnable) {
+ conf.setBoolean(
+ YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
+ }
this.store = new TestFileSystemRMStore(conf);
Assert.assertEquals(store.getNumRetries(), 8);
Assert.assertEquals(store.getRetryInterval(), 900L);
@@ -119,11 +123,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
store.startInternal();
Assert.assertTrue(store.fs != previousFs);
Assert.assertTrue(store.fs.getConf() == store.fsConf);
- if (adminCheckEnable) {
- store.setIsHDFS(true);
- } else {
- store.setIsHDFS(false);
- }
return store;
}