You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/09/20 22:22:09 UTC
[47/50] [abbrv] hadoop git commit: Fix merge conflicts
Fix merge conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5641534e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5641534e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5641534e
Branch: refs/heads/HDDS-4
Commit: 5641534e44d137482a7e5e0fc078ee0d7b684fe0
Parents: f8f02f2
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Jul 31 18:17:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Sep 20 15:13:07 2018 -0700
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 2 -
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 1 -
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 ++-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 5 --
.../common/src/main/resources/ozone-default.xml | 33 +++-----
.../scm/server/StorageContainerManager.java | 8 +-
.../StorageContainerManagerHttpServer.java | 4 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 3 +-
.../src/test/compose/compose-secure/.env | 2 +-
.../test/compose/compose-secure/docker-config | 55 ++++++++++---
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 --------------------
.../apache/hadoop/ozone/MiniOzoneCluster.java | 3 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 22 ++---
14 files changed, 87 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index bce1cd5..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,8 +436,6 @@ public abstract class TestConfigurationFieldsBase {
// Create XML key/value map
LOG_XML.debug("Reading XML property files\n");
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
- // Remove hadoop property set in ozone-default.xml
- xmlKeyValueMap.remove("hadoop.custom.tags");
LOG_XML.debug("\n=====\n");
// Create default configuration variable key/value map
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index adadcd2..6524d4a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -112,5 +112,4 @@ public final class HddsConfigKeys {
public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ ".name";
public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
->>>>>>> HDDS-100. SCM CA: generate public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 4b0e7c8..bc90c1e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -279,10 +279,12 @@ public final class ScmConfigKeys {
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
- "ozone.scm.web.authentication.kerberos.principal";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.scm.web.authentication.kerberos.keytab";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+ "hdds.scm.web.authentication.kerberos.principal";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.web.authentication.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 944605b..cea23cb 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -302,11 +302,6 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
- public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om."
- + "kerberos.keytab.file";
- public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
- + ".kerberos.principal";
-
/**
* There is no need to instantiate this class.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index dd0aca3..18450a9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -433,7 +433,6 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
@@ -443,9 +442,6 @@
</property>
<property>
<name>ozone.om.db.cache.size.mb</name>
-=======
- <name>ozone.ksm.db.cache.size.mb</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>128</value>
<tag>OM, PERFORMANCE</tag>
<description>
@@ -933,7 +929,7 @@
</property>
<property>
- <name>hdds.scm.container.creation.lease.timeout</name>
+ <name>ozone.scm.container.creation.lease.timeout</name>
<value>60s</value>
<tag>OZONE, SCM</tag>
<description>
@@ -987,11 +983,7 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>hdds.container.close.threshold</name>
-=======
- <name>hdds.scm.container.close.threshold</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>0.9f</value>
<tag>OZONE, DATANODE</tag>
<description>
@@ -1123,6 +1115,7 @@
to occur.
</description>
</property>
+
<property>
<name>hdds.lock.suppress.warning.interval.ms</name>
<value>10000</value>
@@ -1132,13 +1125,7 @@
consecutive warnings within this interval.
</description>
</property>
- <property>
- <name>ozone.security.enabled</name>
- <value>false</value>
- <tag> OZONE, SECURITY, FLAG</tag>
- <description>True if security is enabled for ozone. When this property is true, hadoop.security.authentication should be Kerberos.
- </description>
- </property>
+
<property>
<name>hdds.command.status.report.interval</name>
<value>30s</value>
@@ -1206,8 +1193,8 @@
datanode unless the datanode confirms the completion.
</description>
</property>
+
<property>
-<<<<<<< HEAD
<name>hdds.db.profile</name>
<value>SSD</value>
<tag>OZONE, OM, PERFORMANCE, REQUIRED</tag>
@@ -1215,6 +1202,7 @@
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
</property>
+
<property>
<name>hdds.datanode.replication.work.dir</name>
<tag>DATANODE</tag>
@@ -1235,10 +1223,15 @@
</property>
<property>
- <name>ozone.scm.kerberos.keytab.file</name>
-=======
+ <name>ozone.security.enabled</name>
+ <value>false</value>
+ <tag> OZONE, SECURITY, FLAG</tag>
+ <description>True if security is enabled for ozone. When this property is true, hadoop.security.authentication should be Kerberos.
+ </description>
+ </property>
+
+ <property>
<name>hdds.scm.kerberos.keytab.file</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value></value>
<tag> OZONE, SECURITY</tag>
<description> The keytab file used by each SCM daemon to login as its
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 47830e6..03ca9ab 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -433,7 +433,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
-<<<<<<< HEAD
/**
* Create an SCM instance based on the supplied command-line arguments.
*
@@ -444,11 +443,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param args command line arguments.
* @param conf HDDS configuration
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
@VisibleForTesting
public static StorageContainerManager createSCM(
- String[] args, OzoneConfiguration conf) throws IOException {
+ String[] args, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -459,7 +459,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param conf HDDS configuration
* @param printBanner if true, then log a verbose startup message.
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
private static StorageContainerManager createSCM(
String[] args,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index da936ad..41dd89a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index e70e444..d1a1235 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -106,7 +107,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
}
private static StorageContainerManager getScm(OzoneConfiguration conf)
- throws IOException {
+ throws IOException, AuthenticationException {
conf.setBoolean(OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
if(scmStore.getState() != StorageState.INITIALIZED) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
index 3254735..199d52c 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
+OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 360b69a..cdf72ce 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -22,7 +22,7 @@ OZONE-SITE.XML_hdds.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
@@ -32,21 +32,12 @@ OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.C
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.block.access.token.enable=true
-HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
@@ -55,9 +46,51 @@ LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn om scm HTTP testuser
+KERBEROS_KEYTABS=dn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index cc25dbe..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
- /**
- * Never constructed.
- */
- private KSMConfigKeys() {
- }
-
-
- public static final String OZONE_KSM_HANDLER_COUNT_KEY =
- "ozone.ksm.handler.count.key";
- public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
- public static final String OZONE_KSM_ADDRESS_KEY =
- "ozone.ksm.address";
- public static final String OZONE_KSM_BIND_HOST_DEFAULT =
- "0.0.0.0";
- public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
- public static final String OZONE_KSM_HTTP_ENABLED_KEY =
- "ozone.ksm.http.enabled";
- public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
- "ozone.ksm.http-bind-host";
- public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
- "ozone.ksm.https-bind-host";
- public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
- "ozone.ksm.http-address";
- public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
- "ozone.ksm.https-address";
- public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
- public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
- public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
- "ozone.ksm.db.cache.size.mb";
- public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_KSM_USER_MAX_VOLUME =
- "ozone.ksm.user.max.volume";
- public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
- // KSM Default user/group permissions
- public static final String OZONE_KSM_USER_RIGHTS =
- "ozone.ksm.user.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KSM_GROUP_RIGHTS =
- "ozone.ksm.group.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
- "ozone.key.deleting.limit.per.task";
- public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.om.web.authentication.kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
- "ozone.om.web.authentication.kerberos.keytab";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index d13efb4..e2da410 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import java.io.IOException;
@@ -142,7 +143,7 @@ public interface MiniOzoneCluster {
* @throws InterruptedException
*/
void restartStorageContainerManager() throws InterruptedException,
- TimeoutException, IOException;
+ TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 113fea8..2fe499e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -217,7 +217,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
@Override
public void restartStorageContainerManager()
- throws TimeoutException, InterruptedException, IOException {
+ throws TimeoutException, InterruptedException, IOException, AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5641534e/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 3fb5499..14c6800 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -89,10 +89,6 @@ public final class TestSecureOzoneCluster {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
-
- clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
- omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -120,11 +116,12 @@ public final class TestSecureOzoneCluster {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(omKeyTab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -156,7 +153,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
@@ -170,7 +167,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
@@ -189,6 +186,11 @@ public final class TestSecureOzoneCluster {
private void initSCM()
throws IOException, AuthenticationException {
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org