You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/11/29 20:44:17 UTC

[01/50] [abbrv] hadoop git commit: YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda) [Forced Update!]

Repository: hadoop
Updated Branches:
  refs/heads/HDDS-4 278d4b9b7 -> 187bbbe68 (forced update)


YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda)

Change-Id: Iece9b47438357077a53984a820d4d6423f480518


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed87567
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed87567
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed87567

Branch: refs/heads/HDDS-4
Commit: 9ed87567ad0f1c26a263ce6d8fba56d066260c5c
Parents: 579ef4b
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:31:31 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:31:31 2018 -0800

----------------------------------------------------------------------
 .../native/container-executor/impl/modules/fpga/fpga-module.c   | 5 +++--
 .../native/container-executor/impl/modules/gpu/gpu-module.c     | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
index c1a2f83..e947d7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
@@ -141,7 +141,7 @@ void reload_fpga_configuration() {
 /*
  * Format of FPGA request commandline:
  *
- * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y
+ * c-e --module-fpga --excluded_fpgas 0,1,3 --container_id container_x_y
  */
 int handle_fpga_request(update_cgroups_parameters_function func,
     const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_fpga_request(update_cgroups_parameters_function func,
 
   if (!minor_devices) {
      // Minor devices is null, skip following call.
-     fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+     fprintf(ERRORFILE,
+     "--excluded-fpgas is not specified, skip cgroups call.\n");
      goto cleanup;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index 1a1b164..7522338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -141,7 +141,7 @@ void reload_gpu_configuration() {
 /*
  * Format of GPU request commandline:
  *
- * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y
+ * c-e --module-gpu --excluded_gpus 0,1,3 --container_id container_x_y
  */
 int handle_gpu_request(update_cgroups_parameters_func func,
     const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_gpu_request(update_cgroups_parameters_func func,
 
   if (!minor_devices) {
      // Minor devices is null, skip following call.
-     fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+     fprintf(ERRORFILE,
+     "--excluded_gpus is not specified, skip cgroups call.\n");
      goto cleanup;
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[50/50] [abbrv] hadoop git commit: HDDS-804. Block token: Add secret token manager. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-804. Block token: Add secret token manager. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/187bbbe6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/187bbbe6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/187bbbe6

Branch: refs/heads/HDDS-4
Commit: 187bbbe68cc87729f327e5ed614474d1269b8d85
Parents: 87f51d2
Author: Ajay Kumar <aj...@apache.org>
Authored: Thu Nov 29 08:00:41 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:58:55 2018 -0800

----------------------------------------------------------------------
 .../hdds/security/x509/SecurityConfig.java      |   9 +
 .../security/OzoneBlockTokenSecretManager.java  | 191 +++++++
 .../OzoneDelegationTokenSecretManager.java      | 455 +++++++++++++++++
 .../ozone/security/OzoneSecretManager.java      | 498 ++++---------------
 .../TestOzoneBlockTokenSecretManager.java       | 146 ++++++
 .../TestOzoneDelegationTokenSecretManager.java  | 218 ++++++++
 .../ozone/security/TestOzoneSecretManager.java  | 216 --------
 .../apache/hadoop/ozone/om/OzoneManager.java    |  23 +-
 .../security/TestOzoneManagerBlockToken.java    | 251 ++++++++++
 9 files changed, 1371 insertions(+), 636 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index ee20a21..b38ee7c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdds.security.x509;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,6 +78,7 @@ public class SecurityConfig {
   private final Duration certDuration;
   private final String x509SignatureAlgo;
   private final Boolean grpcBlockTokenEnabled;
+  private final int getMaxKeyLength;
   private final String certificateDir;
   private final String certificateFileName;
 
@@ -88,6 +90,9 @@ public class SecurityConfig {
   public SecurityConfig(Configuration configuration) {
     Preconditions.checkNotNull(configuration, "Configuration cannot be null");
     this.configuration = configuration;
+    this.getMaxKeyLength = configuration.getInt(
+        OzoneConfigKeys.OZONE_MAX_KEY_LEN,
+        OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
     this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
     this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
         HDDS_DEFAULT_KEY_ALGORITHM);
@@ -289,4 +294,8 @@ public class SecurityConfig {
       throw new SecurityException("Unknown security provider:" + provider);
     }
   }
+
+  public int getMaxKeyLength() {
+    return this.getMaxKeyLength;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
new file mode 100644
index 0000000..3b833cb
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+/**
+ * SecretManager for Ozone Master block tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneBlockTokenSecretManager extends
+    OzoneSecretManager<OzoneBlockTokenIdentifier> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneBlockTokenSecretManager.class);;
+  // Will be set by grpc clients for individual datanodes.
+  static final Text SERVICE = new Text("HDDS_SERVICE");
+  private final String omCertSerialId;
+
+  /**
+   * Create a secret manager.
+   *
+   * @param conf
+   * @param blockTokenExpirytime token expiry time for expired tokens in
+   * milliseconds
+   */
+  public OzoneBlockTokenSecretManager(OzoneConfiguration conf,
+      long blockTokenExpirytime, String omCertSerialId) {
+    super(conf, blockTokenExpirytime, blockTokenExpirytime, SERVICE, LOG);
+    this.omCertSerialId = omCertSerialId;
+  }
+
+  @Override
+  public OzoneBlockTokenIdentifier createIdentifier() {
+    throw new SecurityException("Ozone block token can't be created "
+        + "without owner and access mode information.");
+  }
+
+  public OzoneBlockTokenIdentifier createIdentifier(String owner,
+      String blockId, EnumSet<AccessModeProto> modes, long maxLength) {
+    return new OzoneBlockTokenIdentifier(owner, blockId, modes,
+        getTokenExpiryTime(), omCertSerialId, maxLength);
+  }
+
+  /**
+   * Generate an block token for specified user, blockId.
+   *
+   * @param user
+   * @param blockId
+   * @param modes
+   * @param maxLength
+   * @return token
+   */
+  public Token<OzoneBlockTokenIdentifier> generateToken(String user,
+      String blockId, EnumSet<AccessModeProto> modes, long maxLength) {
+    OzoneBlockTokenIdentifier tokenIdentifier = createIdentifier(user,
+        blockId, modes, maxLength);
+    if (LOG.isTraceEnabled()) {
+      long expiryTime = tokenIdentifier.getExpiryDate();
+      String tokenId = tokenIdentifier.toString();
+      LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+          expiryTime, tokenId);
+    }
+    return new Token<>(tokenIdentifier.getBytes(),
+        createPassword(tokenIdentifier), tokenIdentifier.getKind(), SERVICE);
+  }
+
+  /**
+   * Generate an block token for current user.
+   *
+   * @param blockId
+   * @param modes
+   * @return token
+   */
+  public Token<OzoneBlockTokenIdentifier> generateToken(String blockId,
+      EnumSet<AccessModeProto> modes, long maxLength) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String userID = (ugi == null ? null : ugi.getShortUserName());
+    return generateToken(userID, blockId, modes, maxLength);
+  }
+
+  @Override
+  public byte[] retrievePassword(OzoneBlockTokenIdentifier identifier)
+      throws InvalidToken {
+    validateToken(identifier);
+    return createPassword(identifier);
+  }
+
+  @Override
+  public long renewToken(Token<OzoneBlockTokenIdentifier> token,
+      String renewer) throws IOException {
+    throw new UnsupportedOperationException("Renew token operation is not " +
+        "supported for ozone block tokens.");
+  }
+
+  @Override
+  public OzoneBlockTokenIdentifier cancelToken(Token<OzoneBlockTokenIdentifier>
+      token, String canceller) throws IOException {
+    throw new UnsupportedOperationException("Cancel token operation is not " +
+        "supported for ozone block tokens.");
+  }
+
+  /**
+   * Find the OzoneBlockTokenInfo for the given token id, and verify that if the
+   * token is not expired.
+   */
+  public boolean validateToken(OzoneBlockTokenIdentifier identifier)
+      throws InvalidToken {
+    long now = Time.now();
+    if (identifier.getExpiryDate() < now) {
+      throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+          "expired, current time: " + Time.formatTime(now) +
+          " expiry time: " + identifier.getExpiryDate());
+    }
+
+    if (!verifySignature(identifier, createPassword(identifier))) {
+      throw new InvalidToken("Tampared/Inavalid token.");
+    }
+    return true;
+  }
+
+  /**
+   * Should be called before this object is used.
+   */
+  @Override
+  public synchronized void start(KeyPair keyPair) throws IOException {
+    super.start(keyPair);
+    removeExpiredKeys();
+  }
+
+  /**
+   * Returns expiry time by adding configured expiry time with current time.
+   *
+   * @return Expiry time.
+   */
+  private long getTokenExpiryTime() {
+    return Time.now() + getTokenRenewInterval();
+  }
+
+  /**
+   * Should be called before this object is used.
+   */
+  @Override
+  public synchronized void stop() throws IOException {
+    super.stop();
+  }
+
+  private synchronized void removeExpiredKeys() {
+    // TODO: handle roll private key/certificate
+    long now = Time.now();
+    for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+        .iterator(); it.hasNext();) {
+      Map.Entry<Integer, OzoneSecretKey> e = it.next();
+      OzoneSecretKey key = e.getValue();
+      if (key.getExpiryDate() < now) {
+        it.remove();
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
new file mode 100644
index 0000000..1b9414b
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
@@ -0,0 +1,455 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * SecretManager for Ozone Master. Responsible for signing identifiers with
+ * private key,
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneDelegationTokenSecretManager<T extends OzoneTokenIdentifier>
+    extends OzoneSecretManager<T> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneDelegationTokenSecretManager.class);
+  private final Map<T, TokenInfo> currentTokens;
+  private final OzoneSecretStore store;
+  private Thread tokenRemoverThread;
+  private final long tokenRemoverScanInterval;
+  /**
+   * If the delegation token update thread holds this lock, it will not get
+   * interrupted.
+   */
+  private Object noInterruptsLock = new Object();
+
+  /**
+   * Create a secret manager.
+   *
+   * @param conf configuration.
+   * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
+   * milliseconds
+   * @param tokenRenewInterval how often the tokens must be renewed in
+   * milliseconds
+   * @param dtRemoverScanInterval how often the tokens are scanned for expired
+   * tokens in milliseconds
+   */
+  public OzoneDelegationTokenSecretManager(OzoneConfiguration conf,
+      long tokenMaxLifetime, long tokenRenewInterval,
+      long dtRemoverScanInterval, Text service) throws IOException {
+    super(conf, tokenMaxLifetime, tokenRenewInterval, service, LOG);
+    currentTokens = new ConcurrentHashMap();
+    this.tokenRemoverScanInterval = dtRemoverScanInterval;
+    this.store = new OzoneSecretStore(conf);
+    loadTokenSecretState(store.loadState());
+  }
+
+  @Override
+  public T createIdentifier() {
+    return (T) T.newInstance();
+  }
+
+  /**
+   * Create new Identifier with given,owner,renwer and realUser.
+   *
+   * @return T
+   */
+  public T createIdentifier(Text owner, Text renewer, Text realUser) {
+    return (T) T.newInstance(owner, renewer, realUser);
+  }
+
+  /**
+   * Returns {@link Token} for given identifier.
+   *
+   * @param owner
+   * @param renewer
+   * @param realUser
+   * @return Token
+   * @throws IOException to allow future exceptions to be added without breaking
+   *                     compatibility
+   */
+  public Token<T> createToken(Text owner, Text renewer, Text realUser)
+      throws IOException {
+    T identifier = createIdentifier(owner, renewer, realUser);
+    updateIdentifierDetails(identifier);
+
+    byte[] password = createPassword(identifier.getBytes(),
+        getCurrentKey().getPrivateKey());
+    addToTokenStore(identifier, password);
+    Token<T> token = new Token<>(identifier.getBytes(), password,
+        identifier.getKind(), getService());
+    if (LOG.isTraceEnabled()) {
+      long expiryTime = identifier.getIssueDate() + getTokenRenewInterval();
+      String tokenId = identifier.toStringStable();
+      LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+          expiryTime, tokenId);
+    }
+    return token;
+  }
+
+  /**
+   * Stores given identifier in token store.
+   *
+   * @param identifier
+   * @param password
+   * @throws IOException
+   */
+  private void addToTokenStore(T identifier, byte[] password)
+      throws IOException {
+    TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
+        + getTokenRenewInterval(), password, identifier.getTrackingId());
+    currentTokens.put(identifier, tokenInfo);
+    store.storeToken(identifier, tokenInfo.getRenewDate());
+  }
+
+  /**
+   * Updates issue date, master key id and sequence number for identifier.
+   *
+   * @param identifier the identifier to validate
+   */
+  private void updateIdentifierDetails(T identifier) {
+    int sequenceNum;
+    long now = Time.monotonicNow();
+    sequenceNum = incrementDelegationTokenSeqNum();
+    identifier.setIssueDate(now);
+    identifier.setMasterKeyId(getCurrentKey().getKeyId());
+    identifier.setSequenceNumber(sequenceNum);
+    identifier.setMaxDate(Time.monotonicNow() + getTokenMaxLifetime());
+  }
+
+  /**
+   * Renew a delegation token.
+   *
+   * @param token the token to renew
+   * @param renewer the full principal name of the user doing the renewal
+   * @return the new expiration time
+   * @throws InvalidToken           if the token is invalid
+   * @throws AccessControlException if the user can't renew token
+   */
+  @Override
+  public synchronized long renewToken(Token<T> token, String renewer)
+      throws IOException {
+    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+    DataInputStream in = new DataInputStream(buf);
+    T id = (T) T.readProtoBuf(in);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
+          formatTokenId(id), currentTokens.size());
+    }
+
+    long now = Time.monotonicNow();
+    if (id.getMaxDate() < now) {
+      throw new InvalidToken(renewer + " tried to renew an expired token "
+          + formatTokenId(id) + " max expiration date: "
+          + Time.formatTime(id.getMaxDate())
+          + " currentTime: " + Time.formatTime(now));
+    }
+    validateToken(id);
+    if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
+      throw new AccessControlException(renewer +
+          " tried to renew a token " + formatTokenId(id)
+          + " without a renewer");
+    }
+    if (!id.getRenewer().toString().equals(renewer)) {
+      throw new AccessControlException(renewer
+          + " tries to renew a token " + formatTokenId(id)
+          + " with non-matching renewer " + id.getRenewer());
+    }
+    OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
+    if (key == null) {
+      throw new InvalidToken("Unable to find master key for keyId="
+          + id.getMasterKeyId()
+          + " from cache. Failed to renew an unexpired token "
+          + formatTokenId(id) + " with sequenceNumber="
+          + id.getSequenceNumber());
+    }
+    byte[] password = createPassword(token.getIdentifier(),
+        key.getPrivateKey());
+
+    long renewTime = Math.min(id.getMaxDate(), now + getTokenRenewInterval());
+    try {
+      addToTokenStore(id, password);
+    } catch (IOException e) {
+      LOG.error("Unable to update token " + id.getSequenceNumber(), e);
+    }
+    return renewTime;
+  }
+
+  /**
+   * Cancel a token by removing it from store and cache.
+   *
+   * @return Identifier of the canceled token
+   * @throws InvalidToken           for invalid token
+   * @throws AccessControlException if the user isn't allowed to cancel
+   */
+  public T cancelToken(Token<T> token, String canceller) throws IOException {
+    T id = (T) T.readProtoBuf(token.getIdentifier());
+    LOG.debug("Token cancellation requested for identifier: {}",
+        formatTokenId(id));
+
+    if (id.getUser() == null) {
+      throw new InvalidToken("Token with no owner " + formatTokenId(id));
+    }
+    String owner = id.getUser().getUserName();
+    Text renewer = id.getRenewer();
+    HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
+    String cancelerShortName = cancelerKrbName.getShortName();
+    if (!canceller.equals(owner)
+        && (renewer == null || renewer.toString().isEmpty()
+        || !cancelerShortName
+        .equals(renewer.toString()))) {
+      throw new AccessControlException(canceller
+          + " is not authorized to cancel the token " + formatTokenId(id));
+    }
+    try {
+      store.removeToken(id);
+    } catch (IOException e) {
+      LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
+    }
+    TokenInfo info = currentTokens.remove(id);
+    if (info == null) {
+      throw new InvalidToken("Token not found " + formatTokenId(id));
+    }
+    return id;
+  }
+
+  @Override
+  public byte[] retrievePassword(T identifier) throws InvalidToken {
+    return validateToken(identifier).getPassword();
+  }
+
+  /**
+   * Checks if TokenInfo for the given identifier exists in database and if the
+   * token is expired.
+   */
+  public TokenInfo validateToken(T identifier) throws InvalidToken {
+    TokenInfo info = currentTokens.get(identifier);
+    if (info == null) {
+      throw new InvalidToken("token " + formatTokenId(identifier)
+          + " can't be found in cache");
+    }
+    long now = Time.monotonicNow();
+    if (info.getRenewDate() < now) {
+      throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+          "expired, current time: " + Time.formatTime(now) +
+          " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+    }
+    if (!verifySignature(identifier, info.getPassword())) {
+      throw new InvalidToken("Tampared/Inavalid token.");
+    }
+    return info;
+  }
+
+  // TODO: handle roll private key/certificate
+  private synchronized void removeExpiredKeys() {
+    long now = Time.monotonicNow();
+    for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+        .iterator(); it.hasNext();) {
+      Map.Entry<Integer, OzoneSecretKey> e = it.next();
+      OzoneSecretKey key = e.getValue();
+      if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
+        if (!key.equals(getCurrentKey())) {
+          it.remove();
+          try {
+            store.removeTokenMasterKey(key);
+          } catch (IOException ex) {
+            LOG.error("Unable to remove master key " + key.getKeyId(), ex);
+          }
+        }
+      }
+    }
+  }
+
+  private void loadTokenSecretState(OzoneManagerSecretState<T> state)
+      throws IOException {
+    LOG.info("Loading token state into token manager.");
+    for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
+      allKeys.putIfAbsent(key.getKeyId(), key);
+      incrementCurrentKeyId();
+    }
+    for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
+      addPersistedDelegationToken(entry.getKey(), entry.getValue());
+    }
+  }
+
+  private void addPersistedDelegationToken(
+      T identifier, long renewDate)
+      throws IOException {
+    if (isRunning()) {
+      // a safety check
+      throw new IOException(
+          "Can't add persisted delegation token to a running SecretManager.");
+    }
+    int keyId = identifier.getMasterKeyId();
+    OzoneSecretKey dKey = allKeys.get(keyId);
+    if (dKey == null) {
+      LOG.warn("No KEY found for persisted identifier "
+          + formatTokenId(identifier));
+      return;
+    }
+
+    PrivateKey privateKey = dKey.getPrivateKey();
+    byte[] password = createPassword(identifier.getBytes(), privateKey);
+    if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
+      setDelegationTokenSeqNum(identifier.getSequenceNumber());
+    }
+    if (currentTokens.get(identifier) == null) {
+      currentTokens.put(identifier, new TokenInfo(renewDate,
+          password, identifier.getTrackingId()));
+    } else {
+      throw new IOException("Same delegation token being added twice: "
+          + formatTokenId(identifier));
+    }
+  }
+
+  /**
+   * Should be called before this object is used.
+   */
+  @Override
+  public synchronized void start(KeyPair keyPair) throws IOException {
+    super.start(keyPair);
+    storeKey(getCurrentKey());
+    removeExpiredKeys();
+    tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
+    tokenRemoverThread.start();
+  }
+
+  private void storeKey(OzoneSecretKey key) throws IOException {
+    store.storeTokenMasterKey(key);
+    if (!allKeys.containsKey(key.getKeyId())) {
+      allKeys.put(key.getKeyId(), key);
+    }
+  }
+
+  public void stopThreads() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Stopping expired delegation token remover thread");
+    }
+    setIsRunning(false);
+
+    if (tokenRemoverThread != null) {
+      synchronized (noInterruptsLock) {
+        tokenRemoverThread.interrupt();
+      }
+      try {
+        tokenRemoverThread.join();
+      } catch (InterruptedException e) {
+        throw new RuntimeException(
+            "Unable to join on token removal thread", e);
+      }
+    }
+  }
+
+  /**
+   * Stops the OzoneDelegationTokenSecretManager.
+   *
+   * @throws IOException
+   */
+  @Override
+  public void stop() throws IOException {
+    super.stop();
+    stopThreads();
+    if (this.store != null) {
+      this.store.close();
+    }
+  }
+
+  /**
+   * Remove expired delegation tokens from cache and persisted store.
+   */
+  private void removeExpiredToken()  {
+    long now = Time.monotonicNow();
+    synchronized (this) {
+      Iterator<Map.Entry<T,
+          TokenInfo>> i = currentTokens.entrySet().iterator();
+      while (i.hasNext()) {
+        Map.Entry<T,
+            TokenInfo> entry = i.next();
+        long renewDate = entry.getValue().getRenewDate();
+        if (renewDate < now) {
+          i.remove();
+          try {
+            store.removeToken(entry.getKey());
+          } catch (IOException e) {
+            if(LOG.isDebugEnabled()) {
+              LOG.debug("Failed to remove expired token {}", entry.getValue());
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private class ExpiredTokenRemover extends Thread {
+    private long lastTokenCacheCleanup;
+
+    @Override
+    public void run() {
+      LOG.info("Starting expired delegation token remover thread, "
+          + "tokenRemoverScanInterval=" + getTokenRemoverScanInterval()
+          / (60 * 1000) + " min(s)");
+      try {
+        while (isRunning()) {
+          long now = Time.monotonicNow();
+          if (lastTokenCacheCleanup + getTokenRemoverScanInterval()
+              < now) {
+            removeExpiredToken();
+            lastTokenCacheCleanup = now;
+          }
+          try {
+            Thread.sleep(Math.min(5000,
+                getTokenRemoverScanInterval())); // 5 seconds
+          } catch (InterruptedException ie) {
+            LOG.error("ExpiredTokenRemover received " + ie);
+          }
+        }
+      } catch (Throwable t) {
+        LOG.error("ExpiredTokenRemover thread received unexpected exception",
+            t);
+        Runtime.getRuntime().exit(-1);
+      }
+    }
+  }
+
+  public long getTokenRemoverScanInterval() {
+    return tokenRemoverScanInterval;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
index 0c84404..01ef8bb 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
@@ -18,8 +18,17 @@
 package org.apache.hadoop.ozone.security;
 
 import com.google.common.base.Preconditions;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.slf4j.Logger;
+
 import java.io.IOException;
 import java.security.InvalidKeyException;
 import java.security.KeyPair;
@@ -27,25 +36,9 @@ import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
 import java.security.Signature;
 import java.security.SignatureException;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
-import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.HadoopKerberosName;
-import org.apache.hadoop.security.token.SecretManager;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * SecretManager for Ozone Master. Responsible for signing identifiers with
@@ -53,33 +46,23 @@ import org.slf4j.LoggerFactory;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public class OzoneSecretManager<T extends OzoneTokenIdentifier>
+public abstract class OzoneSecretManager<T extends TokenIdentifier>
     extends SecretManager<T> {
 
-  private static final Logger LOG = LoggerFactory
-      .getLogger(OzoneSecretManager.class);
+  private final Logger logger;
   /**
    * The name of the Private/Public Key based hashing algorithm.
    */
-  private static final String DEFAULT_SIGNATURE_ALGORITHM = "SHA256withRSA";
+  private final SecurityConfig securityConfig;
   private final long tokenMaxLifetime;
   private final long tokenRenewInterval;
-  private final long tokenRemoverScanInterval;
   private final Text service;
-  private final Map<Integer, OzoneSecretKey> allKeys;
-  private final Map<T, TokenInfo> currentTokens;
-  private final OzoneSecretStore store;
-  private Thread tokenRemoverThread;
   private volatile boolean running;
-  private AtomicInteger tokenSequenceNumber;
   private OzoneSecretKey currentKey;
-  private AtomicInteger currentKeyId;
-  /**
-   * If the delegation token update thread holds this lock, it will not get
-   * interrupted.
-   */
-  private Object noInterruptsLock = new Object();
   private int maxKeyLength;
+  private AtomicInteger currentKeyId;
+  private AtomicInteger tokenSequenceNumber;
+  protected final Map<Integer, OzoneSecretKey> allKeys;
 
   /**
    * Create a secret manager.
@@ -89,100 +72,21 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
    * milliseconds
    * @param tokenRenewInterval how often the tokens must be renewed in
    * milliseconds
-   * @param dtRemoverScanInterval how often the tokens are scanned for expired
-   * tokens in milliseconds
+   * @param service name of service
    */
   public OzoneSecretManager(OzoneConfiguration conf, long tokenMaxLifetime,
-      long tokenRenewInterval, long dtRemoverScanInterval, Text service)
-      throws IOException {
+      long tokenRenewInterval, Text service, Logger logger) {
+    this.securityConfig = new SecurityConfig(conf);
     this.tokenMaxLifetime = tokenMaxLifetime;
     this.tokenRenewInterval = tokenRenewInterval;
-    this.tokenRemoverScanInterval = dtRemoverScanInterval;
-
-    currentTokens = new ConcurrentHashMap();
-    allKeys = new ConcurrentHashMap<>();
     currentKeyId = new AtomicInteger();
     tokenSequenceNumber = new AtomicInteger();
-    this.store = new OzoneSecretStore(conf);
-    loadTokenSecretState(store.loadState());
+    allKeys = new ConcurrentHashMap<>();
     this.service = service;
-    this.maxKeyLength = conf.getInt(OzoneConfigKeys.OZONE_MAX_KEY_LEN,
-        OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
-  }
-
-  @Override
-  public T createIdentifier() {
-    return (T) T.newInstance();
+    this.maxKeyLength = securityConfig.getMaxKeyLength();
+    this.logger = logger;
   }
 
-  /**
-   * Create new Identifier with given,owner,renwer and realUser.
-   *
-   * @return T
-   */
-  public T createIdentifier(Text owner, Text renewer, Text realUser) {
-    return (T) T.newInstance(owner, renewer, realUser);
-  }
-
-  /**
-   * Returns {@link Token} for given identifier.
-   *
-   * @param owner
-   * @param renewer
-   * @param realUser
-   * @return Token
-   * @throws IOException to allow future exceptions to be added without breaking
-   *                     compatibility
-   */
-  public Token<T> createToken(Text owner, Text renewer, Text realUser)
-      throws IOException {
-    T identifier = createIdentifier(owner, renewer, realUser);
-    updateIdentifierDetails(identifier);
-
-    byte[] password = createPassword(identifier.getBytes(),
-        currentKey.getPrivateKey());
-    addToTokenStore(identifier, password);
-    Token<T> token = new Token<>(identifier.getBytes(), password,
-        identifier.getKind(), service);
-    if (LOG.isTraceEnabled()) {
-      long expiryTime = identifier.getIssueDate() + tokenRenewInterval;
-      String tokenId = identifier.toStringStable();
-      LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
-          expiryTime, tokenId);
-    }
-
-    return token;
-  }
-
-  /**
-   * Stores given identifier in token store.
-   *
-   * @param identifier
-   * @param password
-   * @throws IOException
-   */
-  private void addToTokenStore(T identifier, byte[] password)
-      throws IOException {
-    TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
-        + tokenRenewInterval, password, identifier.getTrackingId());
-    currentTokens.put(identifier, tokenInfo);
-    store.storeToken(identifier, tokenInfo.getRenewDate());
-  }
-
-  /**
-   * Updates issue date, master key id and sequence number for identifier.
-   *
-   * @param identifier the identifier to validate
-   */
-  private void updateIdentifierDetails(T identifier) {
-    int sequenceNum;
-    long now = Time.monotonicNow();
-    sequenceNum = incrementDelegationTokenSeqNum();
-    identifier.setIssueDate(now);
-    identifier.setMasterKeyId(currentKey.getKeyId());
-    identifier.setSequenceNumber(sequenceNum);
-    identifier.setMaxDate(Time.monotonicNow() + tokenMaxLifetime);
-  }
 
   /**
    * Compute HMAC of the identifier using the private key and return the output
@@ -196,7 +100,7 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
       throws OzoneSecurityException {
     try {
       Signature rsaSignature = Signature.getInstance(
-          DEFAULT_SIGNATURE_ALGORITHM);
+          getDefaultSignatureAlgorithm());
       rsaSignature.initSign(privateKey);
       rsaSignature.update(identifier);
       return rsaSignature.sign();
@@ -210,22 +114,31 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
 
   @Override
   public byte[] createPassword(T identifier) {
-    LOG.debug("Creating password for identifier: {}, currentKey: {}",
+    logger.debug("Creating password for identifier: {}, currentKey: {}",
         formatTokenId(identifier), currentKey.getKeyId());
     byte[] password = null;
     try {
       password = createPassword(identifier.getBytes(),
           currentKey.getPrivateKey());
     } catch (IOException ioe) {
-      LOG.error("Could not store token {}!!", formatTokenId(identifier),
+      logger.error("Could not store token {}!!", formatTokenId(identifier),
           ioe);
     }
     return password;
   }
 
+  /**
+   * Default implementation for Ozone. Verifies if hash in token is legit.
+   * */
   @Override
   public byte[] retrievePassword(T identifier) throws InvalidToken {
-    return checkToken(identifier).getPassword();
+    byte[] password = createPassword(identifier);
+    // TODO: Revisit this when key/certificate rotation is implemented.
+    // i.e Try all valid keys instead of current key only.
+    if (!verifySignature(identifier, password)) {
+      throw new InvalidToken("Tampared/Inavalid token.");
+    }
+    return password;
   }
 
   /**
@@ -237,52 +150,8 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
    * @throws InvalidToken           if the token is invalid
    * @throws AccessControlException if the user can't renew token
    */
-  public synchronized long renewToken(Token<T> token, String renewer)
-      throws IOException {
-    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
-    DataInputStream in = new DataInputStream(buf);
-    T id = (T) T.readProtoBuf(in);
-    LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
-        formatTokenId(id), currentTokens.size());
-
-    long now = Time.monotonicNow();
-    if (id.getMaxDate() < now) {
-      throw new InvalidToken(renewer + " tried to renew an expired token "
-          + formatTokenId(id) + " max expiration date: "
-          + Time.formatTime(id.getMaxDate())
-          + " currentTime: " + Time.formatTime(now));
-    }
-    checkToken(id);
-    if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
-      throw new AccessControlException(renewer +
-          " tried to renew a token " + formatTokenId(id)
-          + " without a renewer");
-    }
-    if (!id.getRenewer().toString().equals(renewer)) {
-      throw new AccessControlException(renewer
-          + " tries to renew a token " + formatTokenId(id)
-          + " with non-matching renewer " + id.getRenewer());
-    }
-    OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
-    if (key == null) {
-      throw new InvalidToken("Unable to find master key for keyId="
-          + id.getMasterKeyId()
-          + " from cache. Failed to renew an unexpired token "
-          + formatTokenId(id) + " with sequenceNumber="
-          + id.getSequenceNumber());
-    }
-    byte[] password = createPassword(token.getIdentifier(),
-        key.getPrivateKey());
-
-    long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
-    try {
-      addToTokenStore(id, password);
-    } catch (IOException e) {
-      LOG.error("Unable to update token " + id.getSequenceNumber(), e);
-    }
-    return renewTime;
-  }
-
+  public abstract long renewToken(Token<T> token, String renewer)
+      throws IOException;
   /**
    * Cancel a token by removing it from store and cache.
    *
@@ -290,44 +159,8 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
    * @throws InvalidToken           for invalid token
    * @throws AccessControlException if the user isn't allowed to cancel
    */
-  public T cancelToken(Token<T> token, String canceller) throws IOException {
-    T id = (T) T.readProtoBuf(token.getIdentifier());
-    LOG.debug("Token cancellation requested for identifier: {}",
-        formatTokenId(id));
-
-    if (id.getUser() == null) {
-      throw new InvalidToken("Token with no owner " + formatTokenId(id));
-    }
-    String owner = id.getUser().getUserName();
-    Text renewer = id.getRenewer();
-    HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
-    String cancelerShortName = cancelerKrbName.getShortName();
-    if (!canceller.equals(owner)
-        && (renewer == null || renewer.toString().isEmpty()
-        || !cancelerShortName
-        .equals(renewer.toString()))) {
-      throw new AccessControlException(canceller
-          + " is not authorized to cancel the token " + formatTokenId(id));
-    }
-    try {
-      store.removeToken(id);
-    } catch (IOException e) {
-      LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
-    }
-    TokenInfo info = currentTokens.remove(id);
-    if (info == null) {
-      throw new InvalidToken("Token not found " + formatTokenId(id));
-    }
-    return id;
-  }
-
-  public int getCurrentKeyId() {
-    return currentKeyId.get();
-  }
-
-  public void setCurrentKeyId(int keyId) {
-    currentKeyId.set(keyId);
-  }
+  public abstract T cancelToken(Token<T> token, String canceller)
+      throws IOException;
 
   public int incrementCurrentKeyId() {
     return currentKeyId.incrementAndGet();
@@ -346,14 +179,31 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
   }
 
   /**
-   * Validates if given token is valid.
+   * Update the current master key. This is called once by start method before
+   * tokenRemoverThread is created,
+   */
+  private OzoneSecretKey updateCurrentKey(KeyPair keyPair) throws IOException {
+    logger.info("Updating the current master key for generating tokens");
+
+    // TODO: fix me based on the certificate expire time to set the key
+    // expire time.
+    int newCurrentId = incrementCurrentKeyId();
+    OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
+        keyPair, maxKeyLength);
+    currentKey = newKey;
+    return currentKey;
+  }
+
+  /**
+   * Validates if given hash is valid.
    *
    * @param identifier
    * @param password
    */
-  private boolean validateToken(T identifier, byte[] password) {
+  public boolean verifySignature(T identifier, byte[] password) {
     try {
-      Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+      Signature rsaSignature =
+          Signature.getInstance(getDefaultSignatureAlgorithm());
       rsaSignature.initVerify(currentKey.getPublicKey());
       rsaSignature.update(identifier.getBytes());
       return rsaSignature.verify(password);
@@ -363,179 +213,45 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
     }
   }
 
-  /**
-   * Checks if TokenInfo for the given identifier exists in database and if the
-   * token is expired.
-   */
-  public TokenInfo checkToken(T identifier) throws InvalidToken {
-    TokenInfo info = currentTokens.get(identifier);
-    if (info == null) {
-      throw new InvalidToken("token " + formatTokenId(identifier)
-          + " can't be found in cache");
-    }
-    long now = Time.monotonicNow();
-    if (info.getRenewDate() < now) {
-      throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
-          "expired, current time: " + Time.formatTime(now) +
-          " expected renewal time: " + Time.formatTime(info.getRenewDate()));
-    }
-    if (!validateToken(identifier, info.getPassword())) {
-      throw new InvalidToken("Tampared/Inavalid token.");
-    }
-    return info;
-  }
-
-  // TODO: handle roll private key/certificate
-  private synchronized void removeExpiredKeys() {
-    long now = Time.monotonicNow();
-    for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
-        .iterator(); it.hasNext();) {
-      Map.Entry<Integer, OzoneSecretKey> e = it.next();
-      OzoneSecretKey key = e.getValue();
-      if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
-        if (!key.equals(currentKey)) {
-          it.remove();
-          try {
-            store.removeTokenMasterKey(key);
-          } catch (IOException ex) {
-            LOG.error("Unable to remove master key " + key.getKeyId(), ex);
-          }
-        }
-      }
-    }
-  }
-
-  private void loadTokenSecretState(OzoneManagerSecretState<T> state)
-      throws IOException {
-    LOG.info("Loading token state into token manager.");
-    for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
-      allKeys.putIfAbsent(key.getKeyId(), key);
-    }
-    for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
-      addPersistedDelegationToken(entry.getKey(), entry.getValue());
-    }
-  }
-
-  private String formatTokenId(T id) {
+  public String formatTokenId(T id) {
     return "(" + id + ")";
   }
 
-  private void addPersistedDelegationToken(
-      T identifier, long renewDate)
-      throws IOException {
-    if (running) {
-      // a safety check
-      throw new IOException(
-          "Can't add persisted delegation token to a running SecretManager.");
-    }
-    int keyId = identifier.getMasterKeyId();
-    OzoneSecretKey dKey = allKeys.get(keyId);
-    if (dKey == null) {
-      LOG.warn("No KEY found for persisted identifier "
-          + formatTokenId(identifier));
-      return;
-    }
-
-    PrivateKey privateKey = dKey.getPrivateKey();
-    byte[] password = createPassword(identifier.getBytes(), privateKey);
-    if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
-      setDelegationTokenSeqNum(identifier.getSequenceNumber());
-    }
-    if (currentTokens.get(identifier) == null) {
-      currentTokens.put(identifier, new TokenInfo(renewDate,
-          password, identifier.getTrackingId()));
-    } else {
-      throw new IOException("Same delegation token being added twice: "
-          + formatTokenId(identifier));
-    }
-  }
-
   /**
    * Should be called before this object is used.
+   *
+   * @param keyPair
+   * @throws IOException
    */
-  public void startThreads(KeyPair keyPair) throws IOException {
-    Preconditions.checkState(!running);
+  public synchronized void start(KeyPair keyPair) throws IOException {
+    Preconditions.checkState(!isRunning());
     updateCurrentKey(keyPair);
-    removeExpiredKeys();
-    synchronized (this) {
-      running = true;
-      tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
-      tokenRemoverThread.start();
-    }
-  }
-
-  public void stopThreads() {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Stopping expired delegation token remover thread");
-    }
-    running = false;
-
-    if (tokenRemoverThread != null) {
-      synchronized (noInterruptsLock) {
-        tokenRemoverThread.interrupt();
-      }
-      try {
-        tokenRemoverThread.join();
-      } catch (InterruptedException e) {
-        throw new RuntimeException(
-            "Unable to join on token removal thread", e);
-      }
-    }
+    setIsRunning(true);
   }
 
   /**
-   * Stops the OzoneSecretManager.
+   * Stops the OzoneDelegationTokenSecretManager.
    *
    * @throws IOException
    */
-  public void stop() throws IOException {
-    stopThreads();
-    if (this.store != null) {
-      this.store.close();
-    }
+  public synchronized void stop() throws IOException {
+    setIsRunning(false);
   }
 
-  /**
-   * Update the current master key. This is called once by startThreads before
-   * tokenRemoverThread is created,
-   */
-  private void updateCurrentKey(KeyPair keyPair) throws IOException {
-    LOG.info("Updating the current master key for generating tokens");
-
-    // TODO: fix me based on the certificate expire time to set the key
-    // expire time.
-    int newCurrentId = incrementCurrentKeyId();
-    OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
-        keyPair, maxKeyLength);
+  public String getDefaultSignatureAlgorithm() {
+    return securityConfig.getSignatureAlgo();
+  }
 
-    store.storeTokenMasterKey(newKey);
-    if (!allKeys.containsKey(newKey.getKeyId())) {
-      allKeys.put(newKey.getKeyId(), newKey);
-    }
+  public long getTokenMaxLifetime() {
+    return tokenMaxLifetime;
+  }
 
-    synchronized (this) {
-      currentKey = newKey;
-    }
+  public long getTokenRenewInterval() {
+    return tokenRenewInterval;
   }
 
-  /**
-   * Remove expired delegation tokens from cache and persisted store.
-   */
-  private void removeExpiredToken() throws IOException {
-    long now = Time.monotonicNow();
-    synchronized (this) {
-      Iterator<Map.Entry<T,
-          TokenInfo>> i = currentTokens.entrySet().iterator();
-      while (i.hasNext()) {
-        Map.Entry<T,
-            TokenInfo> entry = i.next();
-        long renewDate = entry.getValue().getRenewDate();
-        if (renewDate < now) {
-          i.remove();
-          store.removeToken(entry.getKey());
-        }
-      }
-    }
+  public Text getService() {
+    return service;
   }
 
   /**
@@ -547,52 +263,20 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
     return running;
   }
 
-  /**
-   * Returns expiry time of a token given its identifier.
-   *
-   * @param dtId DelegationTokenIdentifier of a token
-   * @return Expiry time of the token
-   * @throws IOException
-   */
-  public long getTokenExpiryTime(T dtId)
-      throws IOException {
-    TokenInfo info = currentTokens.get(dtId);
-    if (info != null) {
-      return info.getRenewDate();
-    } else {
-      throw new IOException("No delegation token found for this identifier");
-    }
+  public void setIsRunning(boolean val) {
+    running = val;
   }
 
-  private class ExpiredTokenRemover extends Thread {
-    private long lastTokenCacheCleanup;
+  public OzoneSecretKey getCurrentKey() {
+    return currentKey;
+  }
 
-    @Override
-    public void run() {
-      LOG.info("Starting expired delegation token remover thread, "
-          + "tokenRemoverScanInterval=" + tokenRemoverScanInterval
-          / (60 * 1000) + " min(s)");
-      try {
-        while (running) {
-          long now = Time.monotonicNow();
-          if (lastTokenCacheCleanup + tokenRemoverScanInterval
-              < now) {
-            removeExpiredToken();
-            lastTokenCacheCleanup = now;
-          }
-          try {
-            Thread.sleep(Math.min(5000,
-                tokenRemoverScanInterval)); // 5 seconds
-          } catch (InterruptedException ie) {
-            LOG.error("ExpiredTokenRemover received " + ie);
-          }
-        }
-      } catch (Throwable t) {
-        LOG.error("ExpiredTokenRemover thread received unexpected exception",
-            t);
-        Runtime.getRuntime().exit(-1);
-      }
-    }
+  public AtomicInteger getCurrentKeyId() {
+    return currentKeyId;
+  }
+
+  public AtomicInteger getTokenSequenceNumber() {
+    return tokenSequenceNumber;
   }
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
new file mode 100644
index 0000000..4692266
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.security.KeyPair;
+import java.security.Signature;
+import java.security.cert.X509Certificate;
+import java.util.EnumSet;
+
+/**
+ * Test class for {@link OzoneBlockTokenSecretManager}.
+ */
+public class TestOzoneBlockTokenSecretManager {
+
+  private OzoneBlockTokenSecretManager secretManager;
+  private KeyPair keyPair;
+  private X509Certificate x509Certificate;
+  private long expiryTime;
+  private String omCertSerialId;
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneBlockTokenSecretManager.class.getSimpleName());
+
+
+  @Before
+  public void setUp() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+    // Create Ozone Master certificate (SCM CA issued cert) and key store.
+    x509Certificate = KeyStoreTestUtil
+        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+    omCertSerialId = x509Certificate.getSerialNumber().toString();
+    secretManager = new OzoneBlockTokenSecretManager(conf,
+        expiryTime, omCertSerialId);
+    secretManager.start(keyPair);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    secretManager = null;
+  }
+
+  @Test
+  public void testGenerateToken() throws Exception {
+    Token<OzoneBlockTokenIdentifier> token = secretManager.generateToken(
+        "101", EnumSet.allOf(AccessModeProto.class), 100);
+    OzoneBlockTokenIdentifier identifier =
+        OzoneBlockTokenIdentifier.readFieldsProtobuf(new DataInputStream(
+            new ByteArrayInputStream(token.getIdentifier())));
+    // Check basic details.
+    Assert.assertTrue(identifier.getBlockId().equals("101"));
+    Assert.assertTrue(identifier.getAccessModes().equals(EnumSet
+        .allOf(AccessModeProto.class)));
+    Assert.assertTrue(identifier.getOmCertSerialId().equals(omCertSerialId));
+
+    validateHash(token.getPassword(), token.getIdentifier());
+  }
+
+  @Test
+  public void testCreateIdentifierSuccess() throws Exception {
+    OzoneBlockTokenIdentifier btIdentifier = secretManager.createIdentifier(
+        "testUser", "101", EnumSet.allOf(AccessModeProto.class), 100);
+
+    // Check basic details.
+    Assert.assertTrue(btIdentifier.getOwnerId().equals("testUser"));
+    Assert.assertTrue(btIdentifier.getBlockId().equals("101"));
+    Assert.assertTrue(btIdentifier.getAccessModes().equals(EnumSet
+        .allOf(AccessModeProto.class)));
+    Assert.assertTrue(btIdentifier.getOmCertSerialId().equals(omCertSerialId));
+
+    byte[] hash = secretManager.createPassword(btIdentifier);
+    validateHash(hash, btIdentifier.getBytes());
+  }
+
+  /**
+   * Validate hash using public key of KeyPair.
+   * */
+  private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+    Signature rsaSignature =
+        Signature.getInstance(secretManager.getDefaultSignatureAlgorithm());
+    rsaSignature.initVerify(keyPair.getPublic());
+    rsaSignature.update(identifier);
+    Assert.assertTrue(rsaSignature.verify(hash));
+  }
+
+  @Test
+  public void testCreateIdentifierFailure() throws Exception {
+    LambdaTestUtils.intercept(SecurityException.class,
+        "Ozone block token can't be created without owner and access mode "
+            + "information.", () -> {
+          secretManager.createIdentifier();
+        });
+  }
+
+  @Test
+  public void testRenewToken() throws Exception {
+    LambdaTestUtils.intercept(UnsupportedOperationException.class,
+        "Renew token operation is not supported for ozone block" +
+            " tokens.", () -> {
+          secretManager.renewToken(null, null);
+        });
+  }
+
+  @Test
+  public void testCancelToken() throws Exception {
+    LambdaTestUtils.intercept(UnsupportedOperationException.class,
+        "Cancel token operation is not supported for ozone block" +
+            " tokens.", () -> {
+          secretManager.cancelToken(null, null);
+        });
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
new file mode 100644
index 0000000..37ad5ce
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.Signature;
+
+/**
+ * Test class for {@link OzoneDelegationTokenSecretManager}.
+ */
+public class TestOzoneDelegationTokenSecretManager {
+
+  private OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+      secretManager;
+  private SecurityConfig securityConfig;
+  private KeyPair keyPair;
+  private long expiryTime;
+  private Text serviceRpcAdd;
+  private OzoneConfiguration conf;
+  private static final String BASEDIR = GenericTestUtils.getTempPath(
+      TestOzoneDelegationTokenSecretManager.class.getSimpleName());
+  private final static Text TEST_USER = new Text("testUser");
+  private long tokenMaxLifetime = 1000 * 20;
+  private long tokenRemoverScanInterval = 1000 * 20;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new OzoneConfiguration();
+    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+    securityConfig = new SecurityConfig(conf);
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+    serviceRpcAdd = new Text("localhost");
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    secretManager.stop();
+    FileUtils.deleteQuietly(new File(BASEDIR));
+  }
+
+  @Test
+  public void testCreateToken() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    OzoneTokenIdentifier identifier =
+        OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
+    // Check basic details.
+    Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
+    Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
+    Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
+
+    validateHash(token.getPassword(), token.getIdentifier());
+  }
+
+  @Test
+  public void testRenewTokenSuccess() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(10 * 5);
+    long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
+    Assert.assertTrue(renewalTime > 0);
+  }
+
+  /**
+   * Tests failure for mismatch in renewer.
+   */
+  @Test
+  public void testRenewTokenFailure() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    LambdaTestUtils.intercept(AccessControlException.class,
+        "rougeUser tries to renew a token", () -> {
+          secretManager.renewToken(token, "rougeUser");
+        });
+  }
+
+  /**
+   * Tests token renew failure due to max time.
+   */
+  @Test
+  public void testRenewTokenFailureMaxTime() throws Exception {
+    secretManager = createSecretManager(conf, 100,
+        100, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(101);
+    LambdaTestUtils.intercept(IOException.class,
+        "testUser tried to renew an expired token", () -> {
+          secretManager.renewToken(token, TEST_USER.toString());
+        });
+  }
+
+  /**
+   * Tests token renew failure due to renewal time.
+   */
+  @Test
+  public void testRenewTokenFailureRenewalTime() throws Exception {
+    secretManager = createSecretManager(conf, 1000 * 10,
+        10, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(15);
+    LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
+      secretManager.renewToken(token, TEST_USER.toString());
+    });
+  }
+
+  @Test
+  public void testCreateIdentifier() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    OzoneTokenIdentifier identifier = secretManager.createIdentifier();
+    // Check basic details.
+    Assert.assertTrue(identifier.getOwner().equals(new Text("")));
+    Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
+    Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
+  }
+
+  @Test
+  public void testCancelTokenSuccess() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    secretManager.cancelToken(token, TEST_USER.toString());
+  }
+
+  @Test
+  public void testCancelTokenFailure() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.start(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    LambdaTestUtils.intercept(AccessControlException.class,
+        "rougeUser is not authorized to cancel the token", () -> {
+          secretManager.cancelToken(token, "rougeUser");
+        });
+  }
+
+  /**
+   * Validate hash using public key of KeyPair.
+   */
+  private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+    Signature rsaSignature =
+        Signature.getInstance(securityConfig.getSignatureAlgo(),
+            securityConfig.getProvider());
+    rsaSignature.initVerify(keyPair.getPublic());
+    rsaSignature.update(identifier);
+    Assert.assertTrue(rsaSignature.verify(hash));
+  }
+
+  /**
+   * Create instance of {@link OzoneDelegationTokenSecretManager}.
+   */
+  private OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+      createSecretManager(OzoneConfiguration config, long tokenMaxLife,
+      long expiry, long tokenRemoverScanTime) throws IOException {
+    return new OzoneDelegationTokenSecretManager<>(config, tokenMaxLife,
+        expiry, tokenRemoverScanTime, serviceRpcAdd);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
deleted file mode 100644
index e4a8f2b..0000000
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.security;
-
-import java.io.File;
-import java.io.IOException;
-import java.security.KeyPair;
-import java.security.Signature;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for {@link OzoneSecretManager}.
- */
-public class TestOzoneSecretManager {
-
-  private OzoneSecretManager<OzoneTokenIdentifier> secretManager;
-  private SecurityConfig securityConfig;
-  private KeyPair keyPair;
-  private long expiryTime;
-  private Text serviceRpcAdd;
-  private OzoneConfiguration conf;
-  private static final String BASEDIR = GenericTestUtils
-      .getTempPath(TestOzoneSecretManager.class.getSimpleName());
-  private final static Text TEST_USER = new Text("testUser");
-  private long tokenMaxLifetime = 1000 * 20;
-  private long tokenRemoverScanInterval = 1000 * 20;
-
-  @Before
-  public void setUp() throws Exception {
-    conf = new OzoneConfiguration();
-    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
-    securityConfig = new SecurityConfig(conf);
-    // Create Ozone Master key pair.
-    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
-    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-    serviceRpcAdd = new Text("localhost");
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    secretManager.stop();
-    FileUtils.deleteQuietly(new File(BASEDIR));
-  }
-
-  @Test
-  public void testCreateToken() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    OzoneTokenIdentifier identifier =
-        OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
-    // Check basic details.
-    Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
-    Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
-    Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
-
-    validateHash(token.getPassword(), token.getIdentifier());
-  }
-
-  @Test
-  public void testRenewTokenSuccess() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    Thread.sleep(10 * 5);
-    long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
-    Assert.assertTrue(renewalTime > 0);
-  }
-
-  /**
-   * Tests failure for mismatch in renewer.
-   */
-  @Test
-  public void testRenewTokenFailure() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    LambdaTestUtils.intercept(AccessControlException.class,
-        "rougeUser tries to renew a token", () -> {
-          secretManager.renewToken(token, "rougeUser");
-        });
-  }
-
-  /**
-   * Tests token renew failure due to max time.
-   */
-  @Test
-  public void testRenewTokenFailureMaxTime() throws Exception {
-    secretManager = createSecretManager(conf, 100,
-        100, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    Thread.sleep(101);
-    LambdaTestUtils.intercept(IOException.class,
-        "testUser tried to renew an expired token", () -> {
-          secretManager.renewToken(token, TEST_USER.toString());
-        });
-  }
-
-  /**
-   * Tests token renew failure due to renewal time.
-   */
-  @Test
-  public void testRenewTokenFailureRenewalTime() throws Exception {
-    secretManager = createSecretManager(conf, 1000 * 10,
-        10, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    Thread.sleep(15);
-    LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
-      secretManager.renewToken(token, TEST_USER.toString());
-    });
-  }
-
-  @Test
-  public void testCreateIdentifier() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    OzoneTokenIdentifier identifier = secretManager.createIdentifier();
-    // Check basic details.
-    Assert.assertTrue(identifier.getOwner().equals(new Text("")));
-    Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
-    Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
-  }
-
-  @Test
-  public void testCancelTokenSuccess() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    secretManager.cancelToken(token, TEST_USER.toString());
-  }
-
-  @Test
-  public void testCancelTokenFailure() throws Exception {
-    secretManager = createSecretManager(conf, tokenMaxLifetime,
-        expiryTime, tokenRemoverScanInterval);
-    secretManager.startThreads(keyPair);
-    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
-        TEST_USER,
-        TEST_USER);
-    LambdaTestUtils.intercept(AccessControlException.class,
-        "rougeUser is not authorized to cancel the token", () -> {
-          secretManager.cancelToken(token, "rougeUser");
-        });
-  }
-
-  /**
-   * Validate hash using public key of KeyPair.
-   */
-  private void validateHash(byte[] hash, byte[] identifier) throws Exception {
-    Signature rsaSignature =
-        Signature.getInstance(securityConfig.getSignatureAlgo(),
-            securityConfig.getProvider());
-    rsaSignature.initVerify(keyPair.getPublic());
-    rsaSignature.update(identifier);
-    Assert.assertTrue(rsaSignature.verify(hash));
-  }
-
-  /**
-   * Create instance of {@link OzoneSecretManager}.
-   */
-  private OzoneSecretManager<OzoneTokenIdentifier> createSecretManager(
-      OzoneConfiguration config, long tokenMaxLife, long expiry, long
-      tokenRemoverScanTime) throws IOException {
-    return new OzoneSecretManager<>(config, tokenMaxLife,
-        expiry, tokenRemoverScanTime, serviceRpcAdd);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index c6616aa..1e49779 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -49,12 +49,10 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ozone.OzoneSecurityUtil;
 import org.apache.hadoop.ozone.security.OzoneSecurityException;
 import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.ozone.security.OzoneSecretManager;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.audit.AuditAction;
 import org.apache.hadoop.ozone.audit.AuditEventStatus;
@@ -79,6 +77,7 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
 import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -108,9 +107,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
-import java.util.concurrent.TimeUnit;
 
-import static org.apache.hadoop.ozone.security.OzoneSecurityException.ResultCodes.*;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
 import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
@@ -155,8 +152,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
           + StartupOption.HELP.getName() + " ]\n";
   private static final String OM_DAEMON = "om";
   private static boolean securityEnabled = false;
-  private static OzoneSecretManager secretManager;
-  // TO DO: For testing purpose only, remove before commiting
+  private static OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+      secretManager;
   private KeyPair keyPair;
   private CertificateClient certClient;
   private static boolean testSecureOmFlag = false;
@@ -296,9 +293,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   }
 
 
-  private OzoneSecretManager createSecretManager(
-      OzoneConfiguration conf)
-      throws IOException {
+  private OzoneDelegationTokenSecretManager createSecretManager(
+      OzoneConfiguration conf) throws IOException {
     long tokenRemoverScanInterval =
         conf.getTimeDuration(OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY,
             OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT,
@@ -311,8 +307,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
         conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
             OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
             TimeUnit.MILLISECONDS);
-    return new OzoneSecretManager(conf, tokenMaxLifetime, tokenRenewInterval,
-        tokenRemoverScanInterval, omRpcAddressTxt);
+    return new OzoneDelegationTokenSecretManager<>(conf, tokenMaxLifetime,
+        tokenRenewInterval, tokenRemoverScanInterval, omRpcAddressTxt);
   }
 
   private void stopSecretManager() throws IOException {
@@ -327,7 +323,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       try {
         readKeyPair();
         LOG.info("Starting OM secret manager");
-        secretManager.startThreads(keyPair);
+        secretManager.start(keyPair);
       } catch (IOException e) {
         // Inability to start secret manager
         // can't be recovered from.
@@ -351,7 +347,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
           certClient.getPrivateKey(OM_DAEMON));
     } catch (Exception e) {
       throw new OzoneSecurityException("Error reading private file for "
-          + "OzoneManager", e, OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
+          + "OzoneManager", e, OzoneSecurityException
+          .ResultCodes.OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
new file mode 100644
index 0000000..cb7caf3
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneManagerBlockToken {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestOzoneManagerBlockToken.class);
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneManagerBlockToken.class.getSimpleName());
+  private static final String KEYSTORES_DIR =
+      new File(BASEDIR).getAbsolutePath();
+  private static long expiryTime;
+  private static KeyPair keyPair;
+  private static X509Certificate cert;
+  private static final long MAX_LEN = 1000;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    File base = new File(BASEDIR);
+    FileUtil.fullyDelete(base);
+    base.mkdirs();
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    // Create Ozone Master certificate (SCM CA issued cert) and key store.
+    cert = KeyStoreTestUtil
+        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+  }
+
+  @After
+  public void cleanUp() {
+  }
+
+  @Test
+  public void testSignToken() throws GeneralSecurityException, IOException {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+        .getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+        .getAbsolutePath();
+    String trustPassword = "trustPass";
+    String keyStorePassword = "keyStorePass";
+    String keyPassword = "keyPass";
+
+
+    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+        "OzoneMaster", keyPair.getPrivate(), cert);
+
+    // Create trust store and put the certificate in the trust store
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+        cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Sign the OzoneMaster Token with Ozone Master private key
+    PrivateKey privateKey = keyPair.getPrivate();
+    OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+        "testUser", "84940",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+    // Verify a valid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+    // Verify an invalid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    tokenId = new OzoneBlockTokenIdentifier("", "",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+    LOG.info("Unsigned token {} is {}", tokenId,
+        verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+  }
+
+  public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      PrivateKey privateKey) throws NoSuchAlgorithmException,
+      InvalidKeyException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initSign(privateKey);
+    rsaSignature.update(tokenId.getBytes());
+    byte[] signature = rsaSignature.sign();
+    return signature;
+  }
+
+  public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      byte[] signature, Certificate certificate) throws InvalidKeyException,
+      NoSuchAlgorithmException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initVerify(certificate);
+    rsaSignature.update(tokenId.getBytes());
+    boolean isValid = rsaSignature.verify(signature);
+    return isValid;
+  }
+
+  private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+      Mac mac, SecretKey key) {
+    try {
+      mac.init(key);
+    } catch (InvalidKeyException ike) {
+      throw new IllegalArgumentException("Invalid key to HMAC computation",
+          ike);
+    }
+    return mac.doFinal(identifier.getBytes());
+  }
+
+  OzoneBlockTokenIdentifier generateTestToken() {
+    return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+        RandomStringUtils.randomAlphabetic(5),
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+  }
+
+  @Test
+  public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+      CertificateEncodingException, NoSuchProviderException,
+      InvalidKeyException, SignatureException {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordAsym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+    // Create Ozone Master certificate (SCM CA issued cert) and key store
+    X509Certificate omCert;
+    omCert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+        kp, 30, "SHA256withRSA");
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordAsym.add(
+          signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+        duration / testTokenCount);
+
+    startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), omCert);
+    }
+    duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+        + "is {} ns", duration / testTokenCount);
+  }
+
+  @Test
+  public void testSymmetricTokenPerf() {
+    String hmacSHA1 = "HmacSHA1";
+    String hmacSHA256 = "HmacSHA256";
+
+    testSymmetricTokenPerfHelper(hmacSHA1, 64);
+    testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+  }
+
+  public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordSym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyGenerator keyGen;
+    try {
+      keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+      keyGen.init(keyLen);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    Mac mac;
+    try {
+      mac = Mac.getInstance(hmacAlgorithm);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    SecretKey secretKey = keyGen.generateKey();
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordSym.add(
+          signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+        hmacAlgorithm, keyLen, duration / testTokenCount);
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[37/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
new file mode 100644
index 0000000..02d5e28
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests ozone containers via secure grpc/netty.
+ */
+@RunWith(Parameterized.class)
+public class TestSecureOzoneContainer {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestSecureOzoneContainer.class);
+  /**
+   * Set the timeout for every test.
+   */
+  @Rule
+  public Timeout testTimeout = new Timeout(300000);
+
+  @Rule
+  public TemporaryFolder tempFolder = new TemporaryFolder();
+
+  private OzoneConfiguration conf;
+  private SecurityConfig secConfig;
+  private Boolean requireBlockToken;
+  private Boolean hasBlockToken;
+  private Boolean blockTokeExpired;
+
+
+  public TestSecureOzoneContainer(Boolean requireBlockToken,
+      Boolean hasBlockToken, Boolean blockTokenExpired) {
+    this.requireBlockToken = requireBlockToken;
+    this.hasBlockToken = hasBlockToken;
+    this.blockTokeExpired = blockTokenExpired;
+  }
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> blockTokenOptions() {
+    return Arrays.asList(new Object[][] {
+        {true, true, false},
+        {true, true, true},
+        {true, false, false},
+        {false, true, false},
+        {false, false, false}});
+  }
+
+  @Before
+  public void setup() throws IOException{
+    conf = new OzoneConfiguration();
+    String ozoneMetaPath =
+        GenericTestUtils.getTempPath("ozoneMeta");
+    conf.set(OZONE_METADATA_DIRS, ozoneMetaPath);
+
+    secConfig = new SecurityConfig(conf);
+
+  }
+
+  @Test
+  public void testCreateOzoneContainer() throws Exception {
+    LOG.info("Test case: requireBlockToken: {} hasBlockToken: {} " +
+        "blockTokenExpired: {}.", requireBlockToken, hasBlockToken,
+        blockTokeExpired);
+    conf.setBoolean(HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+        requireBlockToken);
+
+    long containerID = ContainerTestHelper.getTestContainerID();
+    OzoneContainer container = null;
+    System.out.println(System.getProperties().getProperty("java.library.path"));
+    try {
+      Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
+      conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
+      conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline
+          .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE)
+          .getValue());
+      conf.setBoolean(
+          OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+
+      DatanodeDetails dn = TestUtils.randomDatanodeDetails();
+      container = new OzoneContainer(dn, conf, null);
+      //Setting scmId, as we start manually ozone container.
+      container.getDispatcher().setScmId(UUID.randomUUID().toString());
+      container.start();
+
+      UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+          "user1",  new String[] {"usergroup"});
+      long expiryDate = (blockTokeExpired) ?
+          Time.now() - 60 * 60 * 2 : Time.now() + 60 * 60 * 24;
+
+      OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+          "testUser", "cid:lud:bcsid",
+          EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+          expiryDate, "1234", 128L);
+
+      int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
+      if (port == 0) {
+        port = secConfig.getConfiguration().getInt(OzoneConfigKeys
+                .DFS_CONTAINER_IPC_PORT,
+            OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+      }
+      InetSocketAddress addr =
+          new InetSocketAddress(dn.getIpAddress(), port);
+
+      Token<OzoneBlockTokenIdentifier> token =
+          new Token(tokenId.getBytes(), new byte[2], tokenId.getKind(),
+              SecurityUtil.buildTokenService(addr));
+      if (hasBlockToken) {
+        ugi.addToken(token);
+      }
+
+      ugi.doAs(new PrivilegedAction<Void>() {
+        @Override
+        public Void run() {
+          try {
+            XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
+            client.connect();
+            createContainerForTesting(client, containerID);
+          } catch (Exception e) {
+            if (requireBlockToken && hasBlockToken && !blockTokeExpired) {
+              fail("Client with BlockToken should succeed when block token is" +
+                  " required.");
+            }
+            if (requireBlockToken && hasBlockToken && blockTokeExpired) {
+              assertTrue("Receive expected exception",
+                  e instanceof SCMSecurityException);
+            }
+            if (requireBlockToken && !hasBlockToken) {
+              assertTrue("Receive expected exception", e instanceof
+                  SCMSecurityException);
+            }
+          }
+          return null;
+        }
+      });
+    } finally {
+      if (container != null) {
+        container.stop();
+      }
+    }
+  }
+
+  public static void createContainerForTesting(XceiverClientSpi client,
+      long containerID) throws Exception {
+    // Create container
+    ContainerProtos.ContainerCommandRequestProto request =
+        ContainerTestHelper.getCreateContainerRequest(
+            containerID, client.getPipeline());
+    ContainerProtos.ContainerCommandResponseProto response =
+        client.sendCommand(request);
+    Assert.assertNotNull(response);
+    Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
deleted file mode 100644
index 5909916..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import java.io.File;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test class for OzoneManagerDelegationToken.
- */
-public class TestOzoneBlockTokenIdentifier {
-
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestOzoneBlockTokenIdentifier.class);
-  private static final String BASEDIR = GenericTestUtils
-      .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
-  private static final String KEYSTORES_DIR =
-      new File(BASEDIR).getAbsolutePath();
-  private static long expiryTime;
-  private static KeyPair keyPair;
-  private static X509Certificate cert;
-
-  @BeforeClass
-  public static void setUp() throws Exception {
-    File base = new File(BASEDIR);
-    FileUtil.fullyDelete(base);
-    base.mkdirs();
-    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-
-    // Create Ozone Master key pair.
-    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
-    // Create Ozone Master certificate (SCM CA issued cert) and key store.
-    cert = KeyStoreTestUtil
-        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
-  }
-
-  @After
-  public void cleanUp() throws Exception {
-    // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
-  }
-
-  @Test
-  public void testSignToken() throws GeneralSecurityException, IOException {
-    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
-        .getAbsolutePath();
-    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
-        .getAbsolutePath();
-    String trustPassword = "trustPass";
-    String keyStorePassword = "keyStorePass";
-    String keyPassword = "keyPass";
-
-
-    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
-        "OzoneMaster", keyPair.getPrivate(), cert);
-
-    // Create trust store and put the certificate in the trust store
-    Map<String, X509Certificate> certs = Collections.singletonMap("server",
-        cert);
-    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
-    // Sign the OzoneMaster Token with Ozone Master private key
-    PrivateKey privateKey = keyPair.getPrivate();
-    OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
-        "testUser", "84940",
-        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
-        expiryTime, cert.getSerialNumber().toString());
-    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
-    // Verify a valid signed OzoneMaster Token with Ozone Master
-    // public key(certificate)
-    boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
-    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
-
-    // Verify an invalid signed OzoneMaster Token with Ozone Master
-    // public key(certificate)
-    tokenId = new OzoneBlockTokenIdentifier("", "",
-        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
-        expiryTime, cert.getSerialNumber().toString());
-    LOG.info("Unsigned token {} is {}", tokenId,
-        verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
-
-  }
-
-  public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
-      PrivateKey privateKey) throws NoSuchAlgorithmException,
-      InvalidKeyException, SignatureException {
-    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
-    rsaSignature.initSign(privateKey);
-    rsaSignature.update(tokenId.getBytes());
-    byte[] signature = rsaSignature.sign();
-    return signature;
-  }
-
-  public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
-      byte[] signature, Certificate certificate) throws InvalidKeyException,
-      NoSuchAlgorithmException, SignatureException {
-    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
-    rsaSignature.initVerify(certificate);
-    rsaSignature.update(tokenId.getBytes());
-    boolean isValid = rsaSignature.verify(signature);
-    return isValid;
-  }
-
-  private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
-      Mac mac, SecretKey key) {
-    try {
-      mac.init(key);
-    } catch (InvalidKeyException ike) {
-      throw new IllegalArgumentException("Invalid key to HMAC computation",
-          ike);
-    }
-    return mac.doFinal(identifier.getBytes());
-  }
-
-  OzoneBlockTokenIdentifier generateTestToken() {
-    return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
-        RandomStringUtils.randomAlphabetic(5),
-        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
-        expiryTime, cert.getSerialNumber().toString());
-  }
-
-  @Test
-  public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
-      CertificateEncodingException, NoSuchProviderException,
-      InvalidKeyException, SignatureException {
-    final int testTokenCount = 1000;
-    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
-    List<byte[]> tokenPasswordAsym = new ArrayList<>();
-    for (int i = 0; i < testTokenCount; i++) {
-      tokenIds.add(generateTestToken());
-    }
-
-    KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
-
-    // Create Ozone Master certificate (SCM CA issued cert) and key store
-    X509Certificate certificate;
-    certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
-        kp, 30, "SHA256withRSA");
-
-    long startTime = Time.monotonicNowNanos();
-    for (int i = 0; i < testTokenCount; i++) {
-      tokenPasswordAsym.add(
-          signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
-    }
-    long duration = Time.monotonicNowNanos() - startTime;
-    LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
-        duration / testTokenCount);
-
-    startTime = Time.monotonicNowNanos();
-    for (int i = 0; i < testTokenCount; i++) {
-      verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
-          certificate);
-    }
-    duration = Time.monotonicNowNanos() - startTime;
-    LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
-        + "is {} ns", duration / testTokenCount);
-  }
-
-  @Test
-  public void testSymmetricTokenPerf() {
-    String hmacSHA1 = "HmacSHA1";
-    String hmacSHA256 = "HmacSHA256";
-
-    testSymmetricTokenPerfHelper(hmacSHA1, 64);
-    testSymmetricTokenPerfHelper(hmacSHA256, 1024);
-  }
-
-  public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
-    final int testTokenCount = 1000;
-    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
-    List<byte[]> tokenPasswordSym = new ArrayList<>();
-    for (int i = 0; i < testTokenCount; i++) {
-      tokenIds.add(generateTestToken());
-    }
-
-    KeyGenerator keyGen;
-    try {
-      keyGen = KeyGenerator.getInstance(hmacAlgorithm);
-      keyGen.init(keyLen);
-    } catch (NoSuchAlgorithmException nsa) {
-      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
-          " algorithm.");
-    }
-
-    Mac mac;
-    try {
-      mac = Mac.getInstance(hmacAlgorithm);
-    } catch (NoSuchAlgorithmException nsa) {
-      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
-          " algorithm.");
-    }
-
-    SecretKey secretKey = keyGen.generateKey();
-
-    long startTime = Time.monotonicNowNanos();
-    for (int i = 0; i < testTokenCount; i++) {
-      tokenPasswordSym.add(
-          signTokenSymmetric(tokenIds.get(i), mac, secretKey));
-    }
-    long duration = Time.monotonicNowNanos() - startTime;
-    LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
-        hmacAlgorithm, keyLen, duration / testTokenCount);
-  }
-
-  // TODO: verify certificate with a trust store
-  public boolean verifyCert(Certificate certificate) {
-    return true;
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[09/50] [abbrv] hadoop git commit: YARN-9069. Fix SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S T.

Posted by xy...@apache.org.
YARN-9069. Fix SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07142f54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07142f54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07142f54

Branch: refs/heads/HDDS-4
Commit: 07142f54a8c7f70857e99c041f3a2a5189c809b5
Parents: a68d766
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 22:02:59 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 22:02:59 2018 +0530

----------------------------------------------------------------------
 .../yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java      | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/07142f54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 163f707..ede0d15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -54,6 +54,8 @@ public class SchedulerInfo {
       this.schedulerName = "Fair Scheduler";
     } else if (rs instanceof FifoScheduler) {
       this.schedulerName = "Fifo Scheduler";
+    } else {
+      this.schedulerName = rs.getClass().getSimpleName();
     }
     this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
     this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[11/50] [abbrv] hadoop git commit: HADOOP-14927. ITestS3GuardTool failures in testDestroyNoBucket(). Contributed by Gabor Bota.

Posted by xy...@apache.org.
HADOOP-14927. ITestS3GuardTool failures in testDestroyNoBucket(). Contributed by Gabor Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb0d3a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb0d3a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb0d3a3

Branch: refs/heads/HDDS-4
Commit: 7eb0d3a32435da110dc9e6004dba8c5c9b082c35
Parents: 184cced
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Nov 28 16:57:12 2018 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Nov 29 09:36:39 2018 -0700

----------------------------------------------------------------------
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java      | 38 ++++++++++++--------
 1 file changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb0d3a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 1316121..aea57a6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -218,6 +218,27 @@ public abstract class S3GuardTool extends Configured implements Tool {
     format.addOptionWithValue(SECONDS_FLAG);
   }
 
+  protected void checkMetadataStoreUri(List<String> paths) throws IOException {
+    // be sure that path is provided in params, so there's no IOoBE
+    String s3Path = "";
+    if(!paths.isEmpty()) {
+      s3Path = paths.get(0);
+    }
+
+    // Check if DynamoDB url is set from arguments.
+    String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
+    if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
+      // If not set, check if filesystem is guarded by creating an
+      // S3AFileSystem and check if hasMetadataStore is true
+      try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
+          S3AFileSystem.newInstance(toUri(s3Path), getConf())){
+        Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
+            "The S3 bucket is unguarded. " + getName()
+                + " can not be used on an unguarded bucket.");
+      }
+    }
+  }
+
   /**
    * Parse metadata store from command line option or HDFS configuration.
    *
@@ -500,20 +521,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
     public int run(String[] args, PrintStream out) throws Exception {
       List<String> paths = parseArgs(args);
       Map<String, String> options = new HashMap<>();
-      String s3Path = paths.get(0);
-
-      // Check if DynamoDB url is set from arguments.
-      String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
-      if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
-        // If not set, check if filesystem is guarded by creating an
-        // S3AFileSystem and check if hasMetadataStore is true
-        try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
-            S3AFileSystem.newInstance(toUri(s3Path), getConf())){
-          Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
-              "The S3 bucket is unguarded. " + getName()
-                  + " can not be used on an unguarded bucket.");
-        }
-      }
+      checkMetadataStoreUri(paths);
 
       String readCap = getCommandFormat().getOptValue(READ_FLAG);
       if (StringUtils.isNotEmpty(readCap)) {
@@ -590,6 +598,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
         throw e;
       }
 
+      checkMetadataStoreUri(paths);
+
       try {
         initMetadataStore(false);
       } catch (FileNotFoundException e) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[49/50] [abbrv] hadoop git commit: HDDS-696. Bootstrap genesis SCM(CA) with self-signed certificate. Contributed by Anu Engineer.

Posted by xy...@apache.org.
HDDS-696. Bootstrap genesis SCM(CA) with self-signed certificate.
Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f51d23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f51d23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f51d23

Branch: refs/heads/HDDS-4
Commit: 87f51d23d9899e46ac31058b726b272f1a7880ba
Parents: 9b9b8e4
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Nov 27 15:02:07 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:58:54 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |  57 ++-
 .../hdds/security/x509/SecurityConfig.java      |  84 ++++-
 .../authority/CertificateServer.java            |  11 +-
 .../certificate/authority/DefaultCAServer.java  | 373 +++++++++++++++++++
 .../certificate/client/CertificateClient.java   |  18 +-
 .../certificate/utils/CertificateCodec.java     | 280 ++++++++++++++
 .../x509/certificate/utils/package-info.java    |  22 ++
 .../certificates/CertificateSignRequest.java    | 245 ------------
 .../certificates/SelfSignedCertificate.java     | 212 -----------
 .../x509/certificates/package-info.java         |  22 --
 .../utils/CertificateSignRequest.java           | 245 ++++++++++++
 .../utils/SelfSignedCertificate.java            | 238 ++++++++++++
 .../x509/certificates/utils/package-info.java   |  22 ++
 .../security/x509/keys/HDDSKeyGenerator.java    |  32 +-
 .../security/x509/keys/HDDSKeyPEMWriter.java    | 255 -------------
 .../hdds/security/x509/keys/KeyCodec.java       | 337 +++++++++++++++++
 .../apache/hadoop/ozone/common/StorageInfo.java |   2 +-
 .../authority/TestDefaultCAServer.java          | 118 ++++++
 .../certificate/authority/package-info.java     |  22 ++
 .../certificate/utils/TestCertificateCodec.java | 218 +++++++++++
 .../TestCertificateSignRequest.java             |  19 +-
 .../x509/certificates/TestRootCertificate.java  |  46 +--
 .../x509/certificates/package-info.java         |   4 +-
 .../x509/keys/TestHDDSKeyPEMWriter.java         | 216 -----------
 .../hdds/security/x509/keys/TestKeyCodec.java   | 216 +++++++++++
 .../hadoop/utils/db/TestDBStoreBuilder.java     |   5 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    |  15 +-
 27 files changed, 2267 insertions(+), 1067 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 553f4aa..a02152d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -1,19 +1,18 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
  * <p>
  * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
  */
 package org.apache.hadoop.hdds;
 
@@ -25,61 +24,45 @@ import org.apache.hadoop.utils.db.DBProfile;
  */
 public final class HddsConfigKeys {
 
-  /**
-   * Do not instantiate.
-   */
-  private HddsConfigKeys() {
-  }
-
   public static final String HDDS_HEARTBEAT_INTERVAL =
       "hdds.heartbeat.interval";
   public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT =
       "30s";
-
   public static final String HDDS_NODE_REPORT_INTERVAL =
       "hdds.node.report.interval";
   public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
       "60s";
-
   public static final String HDDS_CONTAINER_REPORT_INTERVAL =
       "hdds.container.report.interval";
   public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
       "60s";
-
   public static final String HDDS_PIPELINE_REPORT_INTERVAL =
           "hdds.pipeline.report.interval";
   public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
           "60s";
-
   public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
       "hdds.command.status.report.interval";
   public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
       "60s";
-
   public static final String HDDS_CONTAINER_ACTION_MAX_LIMIT =
       "hdds.container.action.max.limit";
   public static final int HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT =
       20;
-
   public static final String HDDS_PIPELINE_ACTION_MAX_LIMIT =
       "hdds.pipeline.action.max.limit";
   public static final int HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT =
       20;
-
   // Configuration to allow volume choosing policy.
   public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
       "hdds.datanode.volume.choosing.policy";
-
   // DB Profiles used by ROCKDB instances.
   public static final String HDDS_DB_PROFILE = "hdds.db.profile";
   public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
-
   // Once a container usage crosses this threshold, it is eligible for
   // closing.
   public static final String HDDS_CONTAINER_CLOSE_THRESHOLD =
       "hdds.container.close.threshold";
   public static final float HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
-
   public static final String HDDS_SCM_CHILLMODE_ENABLED =
       "hdds.scm.chillmode.enabled";
   public static final boolean HDDS_SCM_CHILLMODE_ENABLED_DEFAULT = true;
@@ -97,11 +80,9 @@ public final class HddsConfigKeys {
   public static final String HDDS_SCM_CHILLMODE_THRESHOLD_PCT =
       "hdds.scm.chillmode.threshold.pct";
   public static final double HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT = 0.99;
-
   public static final String HDDS_LOCK_MAX_CONCURRENCY =
       "hdds.lock.max.concurrency";
   public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
-
   // This configuration setting is used as a fallback location by all
   // Ozone/HDDS services for their metadata. It is useful as a single
   // config point for test/PoC clusters.
@@ -121,7 +102,6 @@ public final class HddsConfigKeys {
   public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
   public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
   public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
-
   // TODO : Talk to StorageIO classes and see if they can return a secure
   // storage location for each node.
   public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
@@ -131,7 +111,6 @@ public final class HddsConfigKeys {
   public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
       + ".name";
   public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
-
   /**
    * Maximum duration of certificates issued by SCM including Self-Signed Roots.
    * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
@@ -140,12 +119,20 @@ public final class HddsConfigKeys {
   public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
   // Limit Certificate duration to a max value of 5 years.
   public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
-
   public static final String HDDS_X509_SIGNATURE_ALGO =
       "hdds.x509.signature.algorithm";
   public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
-
-  public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED = "hdds.grpc.block" +
-      ".token.enabled";
+  public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED =
+      "hdds.grpc.block.token.enabled";
   public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
+  public static final String HDDS_X509_DIR_NAME = "hdds.x509.dir.name";
+  public static final String HDDS_X509_DIR_NAME_DEFAULT = "certs";
+  public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
+  public static final String HDDS_X509_FILE_NAME_DEFAULT = "certificate.crt";
+
+  /**
+   * Do not instantiate.
+   */
+  private HddsConfigKeys() {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 2826e55..ee20a21 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -46,6 +46,10 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_FILE_NAME_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
@@ -54,7 +58,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 
 /**
  * A class that deals with all Security related configs in HDDS.
- *
+ * <p>
  * This class allows security configs to be read and used consistently across
  * all of security related code base.
  */
@@ -73,6 +77,8 @@ public class SecurityConfig {
   private final Duration certDuration;
   private final String x509SignatureAlgo;
   private final Boolean grpcBlockTokenEnabled;
+  private final String certificateDir;
+  private final String certificateFileName;
 
   /**
    * Constructs a SecurityConfig.
@@ -108,6 +114,10 @@ public class SecurityConfig {
     this.certDuration = Duration.parse(durationString);
     this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
         HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+    this.certificateDir = this.configuration.get(HDDS_X509_DIR_NAME,
+        HDDS_X509_DIR_NAME_DEFAULT);
+    this.certificateFileName = this.configuration.get(HDDS_X509_FILE_NAME,
+        HDDS_X509_FILE_NAME_DEFAULT);
 
     this.grpcBlockTokenEnabled = this.configuration.getBoolean(
         HDDS_GRPC_BLOCK_TOKEN_ENABLED,
@@ -127,8 +137,17 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the public key file name, This is used for storing the public
-   * keys on disk.
+   * Returns the Standard Certificate file name.
+   *
+   * @return String - Name of the Certificate File.
+   */
+  public String getCertificateFileName() {
+    return certificateFileName;
+  }
+
+  /**
+   * Returns the public key file name, This is used for storing the public keys
+   * on disk.
    *
    * @return String, File name used for public keys.
    */
@@ -137,8 +156,8 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the private key file name.This is used for storing the private
-   * keys on disk.
+   * Returns the private key file name.This is used for storing the private keys
+   * on disk.
    *
    * @return String, File name used for private keys.
    */
@@ -149,16 +168,47 @@ public class SecurityConfig {
   /**
    * Returns the File path to where keys are stored.
    *
-   * @return String Key location.
+   * @return path Key location.
    */
   public Path getKeyLocation() {
     return Paths.get(metadatDir, keyDir);
   }
 
   /**
+   * Returns the File path to where keys are stored with an additional component
+   * name inserted in between.
+   *
+   * @param component - Component Name - String.
+   * @return Path location.
+   */
+  public Path getKeyLocation(String component) {
+    return Paths.get(metadatDir, component, keyDir);
+  }
+
+  /**
+   * Returns the File path to where keys are stored.
+   *
+   * @return path Key location.
+   */
+  public Path getCertificateLocation() {
+    return Paths.get(metadatDir, certificateDir);
+  }
+
+  /**
+   * Returns the File path to where keys are stored with an addition component
+   * name inserted in between.
+   *
+   * @param component - Component Name - String.
+   * @return Path location.
+   */
+  public Path getCertificateLocation(String component) {
+    return Paths.get(metadatDir, component, certificateDir);
+  }
+
+  /**
    * Gets the Key Size, The default key size is 2048, since the default
-   * algorithm used is RSA. User can change this by setting the "hdds.key
-   * .len" in configuration.
+   * algorithm used is RSA. User can change this by setting the "hdds.key.len"
+   * in configuration.
    *
    * @return key size.
    */
@@ -177,8 +227,8 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the Key generation Algorithm used.  User can change this by
-   * setting the "hdds.key.algo" in configuration.
+   * Returns the Key generation Algorithm used.  User can change this by setting
+   * the "hdds.key.algo" in configuration.
    *
    * @return String Algo.
    */
@@ -188,8 +238,8 @@ public class SecurityConfig {
 
   /**
    * Returns the X.509 Signature Algorithm used. This can be changed by setting
-   * "hdds.x509.signature.algorithm" to the new name. The default algorithm
-   * is SHA256withRSA.
+   * "hdds.x509.signature.algorithm" to the new name. The default algorithm is
+   * SHA256withRSA.
    *
    * @return String
    */
@@ -207,11 +257,11 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the maximum length a certificate can be valid in SCM. The
-   * default value is 5 years. This can be changed by setting
-   * "hdds.x509.max.duration" in configuration. The formats accepted are
-   * based on the ISO-8601 duration format PnDTnHnMn.nS
-   *
+   * Returns the maximum length a certificate can be valid in SCM. The default
+   * value is 5 years. This can be changed by setting "hdds.x509.max.duration"
+   * in configuration. The formats accepted are based on the ISO-8601 duration
+   * format PnDTnHnMn.nS
+   * <p>
    * Default value is 5 years and written as P1865D.
    *
    * @return Duration.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index 5b5deef..ee685c8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -20,10 +20,12 @@
 package org.apache.hadoop.hdds.security.x509.certificate.authority;
 
 import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
 import org.bouncycastle.cert.X509CertificateHolder;
 
+import java.io.IOException;
+import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.concurrent.Future;
 
@@ -47,16 +49,19 @@ public interface CertificateServer {
    * Returns the CA Certificate for this CA.
    *
    * @return X509CertificateHolder - Certificate for this CA.
-   * @throws SCMSecurityException -- usually thrown if this CA is not
+   * @throws CertificateException - usually thrown if this CA is not
    *                              initialized.
+   * @throws IOException - on Error.
    */
   X509CertificateHolder getCACertificate()
-      throws SCMSecurityException;
+      throws CertificateException, IOException;
 
   /**
    * Request a Certificate based on Certificate Signing Request.
    *
    * @param csr - Certificate Signing Request.
+   * @param approver - An Enum which says what kind of approval process to
+   * follow.
    * @return A future that will have this certificate when this request is
    * approved.
    * @throws SCMSecurityException - on Error.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
new file mode 100644
index 0000000..f6e6e5c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.concurrent.Future;
+import java.util.function.Consumer;
+
+/**
+ * The default CertificateServer used by SCM. This has no dependencies on any
+ * external system, this allows us to bootstrap a CertificateServer from
+ * Scratch.
+ * <p>
+ * Details =======
+ * <p>
+ * The Default CA server is one of the many possible implementations of an SCM
+ * Certificate Authority.
+ * <p>
+ * A certificate authority needs the Root Certificates and its private key to
+ * operate.  The init function of the DefaultCA Server detects four possible
+ * states the System can be in.
+ * <p>
+ * 1.  Success - This means that the expected Certificates and Keys are in
+ * place, and the CA was able to read those files into memory.
+ * <p>
+ * 2. Missing Keys - This means that private keys are missing. This is an error
+ * state which SCM CA cannot recover from. The cluster might have been
+ * initialized earlier and for some reason, we are not able to find the private
+ * keys for the CA. Eventually we will have 2 ways to recover from this state,
+ * first one is to copy the SCM CA private keys from a backup. Second one is to
+ * rekey the whole cluster. Both of these are improvements we will support in
+ * future.
+ * <p>
+ * 3. Missing Certificate - Similar to Missing Keys, but the root certificates
+ * are missing.
+ * <p>
+ * 4. Initialize - We don't have keys or certificates. DefaultCA assumes that
+ * this is a system bootup and will generate the keys and certificates
+ * automatically.
+ * <p>
+ * The init() follows the following logic,
+ * <p>
+ * 1. Compute the Verification Status -- Success, Missing Keys, Missing Certs or
+ * Initialize.
+ * <p>
+ * 2. ProcessVerificationStatus - Returns a Lambda, based on the Verification
+ * Status.
+ * <p>
+ * 3. Invoke the Lambda function.
+ * <p>
+ * At the end of the init function, we have functional CA. This function can be
+ * invoked as many times since we will regenerate the keys and certs only if
+ * both of them are missing.
+ */
+public class DefaultCAServer implements CertificateServer {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DefaultCAServer.class);
+  private final String subject;
+  private final String clusterID;
+  private final String scmID;
+  private String componentName = Paths.get("scm", "ca").toString();
+  private Path caKeysPath;
+  private Path caRootX509Path;
+  private SecurityConfig config;
+
+  /**
+   * Create an Instance of DefaultCAServer.
+   *
+   * @param subject - String Subject
+   * @param clusterID - String ClusterID
+   * @param scmID - String SCMID.
+   */
+  public DefaultCAServer(String subject, String clusterID, String scmID) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+  }
+
+  @Override
+  public void init(SecurityConfig securityConfig, CAType type)
+      throws SCMSecurityException {
+    caKeysPath = securityConfig.getKeyLocation(componentName);
+    caRootX509Path = securityConfig.getCertificateLocation(componentName);
+    this.config = securityConfig;
+
+    /* In future we will spilt this code to have different kind of CAs.
+     * Right now, we have only self-signed CertificateServer.
+     */
+
+    if (type == CAType.SELF_SIGNED_CA) {
+      VerificationStatus status = verifySelfSignedCA(securityConfig);
+      Consumer<SecurityConfig> caInitializer =
+          processVerificationStatus(status);
+      caInitializer.accept(securityConfig);
+      return;
+    }
+
+    LOG.error("We support only Self-Signed CAs for now.");
+    throw new IllegalStateException("Not implemented functionality requested.");
+  }
+
+  @Override
+  public X509CertificateHolder getCACertificate() throws
+      CertificateException, IOException {
+    CertificateCodec certificateCodec =
+        new CertificateCodec(config, componentName);
+    return certificateCodec.readCertificate();
+  }
+
+  @Override
+  public Future<X509CertificateHolder> requestCertificate(
+      CertificateSignRequest csr, CertificateApprover approver)
+      throws SCMSecurityException {
+    return null;
+  }
+
+  @Override
+  public Future<Boolean> revokeCertificate(X509Certificate certificate,
+      CertificateApprover approver) throws SCMSecurityException {
+    return null;
+  }
+
+  /**
+   * Generates a Self Signed CertificateServer. These are the steps in
+   * generating a Self-Signed CertificateServer.
+   * <p>
+   * 1. Generate a Private/Public Key Pair. 2. Persist to a protected location.
+   * 3. Generate a SelfSigned Root CertificateServer certificate.
+   *
+   * @param securityConfig - Config.
+   */
+  private void generateSelfSignedCA(SecurityConfig securityConfig) throws
+      NoSuchAlgorithmException, NoSuchProviderException, IOException {
+    KeyPair keyPair = generateKeys(securityConfig);
+    generateRootCertificate(securityConfig, keyPair);
+  }
+
+  /**
+   * Verify Self-Signed CertificateServer. 1. Check if the Certificate exist. 2.
+   * Check if the key pair exists.
+   *
+   * @param securityConfig -- Config
+   * @return Verification Status
+   */
+  private VerificationStatus verifySelfSignedCA(SecurityConfig securityConfig) {
+    /*
+    The following is the truth table for the States.
+    True means we have that file False means it is missing.
+    +--------------+--------+--------+--------------+
+    | Certificates |  Keys  | Result |   Function   |
+    +--------------+--------+--------+--------------+
+    | True         | True   | True   | Success      |
+    | False        | False  | True   | Initialize   |
+    | True         | False  | False  | Missing Key  |
+    | False        | True   | False  | Missing Cert |
+    +--------------+--------+--------+--------------+
+
+    This truth table maps to ~(certs xor keys) or certs == keys
+     */
+    boolean keyStatus = checkIfKeysExist();
+    boolean certStatus = checkIfCertificatesExist();
+
+    if ((certStatus == keyStatus) && (certStatus)) {
+      return VerificationStatus.SUCCESS;
+    }
+
+    if ((certStatus == keyStatus) && (!certStatus)) {
+      return VerificationStatus.INITIALIZE;
+    }
+
+    // At this point certStatus is not equal to keyStatus.
+    if (certStatus) {
+      return VerificationStatus.MISSING_KEYS;
+    }
+
+    return VerificationStatus.MISSING_CERTIFICATE;
+  }
+
+  /**
+   * Returns Keys status.
+   *
+   * @return True if the key files exist.
+   */
+  private boolean checkIfKeysExist() {
+    if (!Files.exists(caKeysPath)) {
+      return false;
+    }
+
+    if (!Files.exists(Paths.get(caKeysPath.toString(),
+        this.config.getPrivateKeyFileName()))) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Returns certificate Status.
+   *
+   * @return True if the Certificate files exist.
+   */
+  private boolean checkIfCertificatesExist() {
+    if (!Files.exists(caRootX509Path)) {
+      return false;
+    }
+    if (!Files.exists(Paths.get(caRootX509Path.toString(),
+        this.config.getCertificateFileName()))) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Based on the Status of the verification, we return a lambda that gets
+   * executed by the init function of the CA.
+   *
+   * @param status - Verification Status.
+   */
+  @VisibleForTesting
+  Consumer<SecurityConfig> processVerificationStatus(
+      VerificationStatus status) {
+    Consumer<SecurityConfig> consumer = null;
+    switch (status) {
+    case SUCCESS:
+      consumer = (arg) -> LOG.info("CertificateServer validation is " +
+          "successful");
+      break;
+    case MISSING_KEYS:
+      consumer = (arg) -> {
+        LOG.error("We have found the Certificate for this CertificateServer, " +
+            "but keys used by this CertificateServer is missing. This is a " +
+            "non-recoverable error. Please restart the system after locating " +
+            "the Keys used by the CertificateServer.");
+        LOG.error("Exiting due to unrecoverable CertificateServer error.");
+        throw new IllegalStateException("Missing Keys, cannot continue.");
+      };
+      break;
+    case MISSING_CERTIFICATE:
+      consumer = (arg) -> {
+        LOG.error("We found the keys, but the root certificate for this " +
+            "CertificateServer is missing. Please restart SCM after locating " +
+            "the " +
+            "Certificates.");
+        LOG.error("Exiting due to unrecoverable CertificateServer error.");
+        throw new IllegalStateException("Missing Root Certs, cannot continue.");
+      };
+      break;
+    case INITIALIZE:
+      consumer = (arg) -> {
+        try {
+          generateSelfSignedCA(arg);
+        } catch (NoSuchProviderException | NoSuchAlgorithmException
+            | IOException e) {
+          LOG.error("Unable to initialize CertificateServer.", e);
+        }
+        VerificationStatus newStatus = verifySelfSignedCA(arg);
+        if (newStatus != VerificationStatus.SUCCESS) {
+          LOG.error("Unable to initialize CertificateServer, failed in " +
+              "verification.");
+        }
+      };
+      break;
+    default:
+      /* Make CheckStyle happy */
+      break;
+    }
+    return consumer;
+  }
+
+  /**
+   * Generates a KeyPair for the Certificate.
+   *
+   * @param securityConfig - SecurityConfig.
+   * @return Key Pair.
+   * @throws NoSuchProviderException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws IOException              - on Error.
+   */
+  private KeyPair generateKeys(SecurityConfig securityConfig)
+      throws NoSuchProviderException, NoSuchAlgorithmException, IOException {
+    HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig);
+    KeyPair keys = keyGenerator.generateKey();
+    KeyCodec keyPEMWriter = new KeyCodec(securityConfig,
+        componentName);
+    keyPEMWriter.writeKey(keys);
+    return keys;
+  }
+
+  /**
+   * Generates a self-signed Root Certificate for CA.
+   *
+   * @param securityConfig - SecurityConfig
+   * @param key - KeyPair.
+   * @throws IOException          - on Error.
+   * @throws SCMSecurityException - on Error.
+   */
+  private void generateRootCertificate(SecurityConfig securityConfig,
+      KeyPair key) throws IOException, SCMSecurityException {
+    Preconditions.checkNotNull(this.config);
+    LocalDate beginDate = LocalDate.now().atStartOfDay().toLocalDate();
+    LocalDateTime temp = LocalDateTime.of(beginDate, LocalTime.MIDNIGHT);
+    LocalDate endDate =
+        temp.plus(securityConfig.getMaxCertificateDuration()).toLocalDate();
+    X509CertificateHolder selfSignedCertificate =
+        SelfSignedCertificate
+            .newBuilder()
+            .setSubject(this.subject)
+            .setScmID(this.scmID)
+            .setClusterID(this.clusterID)
+            .setBeginDate(beginDate)
+            .setEndDate(endDate)
+            .makeCA()
+            .setConfiguration(securityConfig.getConfiguration())
+            .setKey(key)
+            .build();
+
+    CertificateCodec certCodec =
+        new CertificateCodec(config, componentName);
+    certCodec.writeCertificate(selfSignedCertificate);
+  }
+
+  /**
+   * This represents the verification status of the CA. Based on this enum
+   * appropriate action is taken in the Init.
+   */
+  @VisibleForTesting
+  enum VerificationStatus {
+    SUCCESS, /* All artifacts needed by CertificateServer is present */
+    MISSING_KEYS, /* Private key is missing, certificate Exists.*/
+    MISSING_CERTIFICATE, /* Keys exist, but root certificate missing.*/
+    INITIALIZE /* All artifacts are missing, we should init the system. */
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index bfc0576..e33c9b6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
-import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
 import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
 
 import java.io.InputStream;
@@ -64,7 +64,7 @@ public interface CertificateClient {
 
   /**
    * Verifies if this certificate is part of a trusted chain.
-   *
+   * @param certificate - certificate.
    * @return true if it trusted, false otherwise.
    */
   boolean verifyCertificate(X509Certificate certificate);
@@ -74,7 +74,9 @@ public interface CertificateClient {
    * key.
    *
    * @param stream - Data stream to sign.
+   * @param component - name of the component.
    * @return byte array - containing the signature.
+   * @throws CertificateException - on Error.
    */
   byte[] signDataStream(InputStream stream, String component)
       throws CertificateException;
@@ -82,6 +84,7 @@ public interface CertificateClient {
   /**
    * Verifies a digital Signature, given the signature and the certificate of
    * the signer.
+   *
    * @param stream - Data Stream.
    * @param signature - Byte Array containing the signature.
    * @param cert - Certificate of the Signer.
@@ -123,7 +126,7 @@ public interface CertificateClient {
    *
    * @param key - private key
    * @param component - name of the component.
-   * @throws CertificateException
+   * @throws CertificateException - on Error.
    */
   void storePrivateKey(PrivateKey key, String component)
       throws CertificateException;
@@ -132,7 +135,8 @@ public interface CertificateClient {
    * Stores the public key of a specified component.
    *
    * @param key - public key
-   * @throws CertificateException
+   * @param component - name of the component.
+   * @throws CertificateException - on Error.
    */
   void storePublicKey(PublicKey key, String component)
       throws CertificateException;
@@ -142,7 +146,7 @@ public interface CertificateClient {
    *
    * @param certificate - X509 Certificate
    * @param component - Name of the component.
-   * @throws CertificateException
+   * @throws CertificateException - on Error.
    */
   void storeCertificate(X509Certificate certificate, String component)
       throws CertificateException;
@@ -152,7 +156,7 @@ public interface CertificateClient {
    *
    * @param certStore - Cert Store.
    * @param component - Trust Chain.
-   * @throws CertificateException
+   * @throws CertificateException - on Error.
    */
   void storeTrustChain(CertStore certStore,
       String component) throws CertificateException;
@@ -162,7 +166,7 @@ public interface CertificateClient {
    *
    * @param certificates - List of Certificates.
    * @param component - String component.
-   * @throws CertificateException
+   * @throws CertificateException - on Error.
    */
   void storeTrustChain(List<X509Certificate> certificates,
       String component) throws CertificateException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
new file mode 100644
index 0000000..b2a37b5
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * A class used to read and write X.509 certificates  PEM encoded Streams.
+ */
+public class CertificateCodec {
+  public static final String BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
+  public static final String END_CERT = "-----END CERTIFICATE-----";
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CertificateCodec.class);
+  private static final JcaX509CertificateConverter CERTIFICATE_CONVERTER
+      = new JcaX509CertificateConverter();
+  private final SecurityConfig securityConfig;
+  private final Path location;
+  private Set<PosixFilePermission> permissionSet =
+      Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+          .collect(Collectors.toSet());
+  /**
+   * Creates an CertificateCodec.
+   *
+   * @param config - Security Config.
+   * @param component - Component String.
+   */
+  public CertificateCodec(SecurityConfig config, String component) {
+    this.securityConfig = config;
+    this.location = securityConfig.getCertificateLocation(component);
+  }
+
+  /**
+   * Creates an CertificateCodec.
+   *
+   * @param configuration - Configuration
+   */
+  public CertificateCodec(Configuration configuration) {
+    Preconditions.checkNotNull(configuration, "Config cannot be null");
+    this.securityConfig = new SecurityConfig(configuration);
+    this.location = securityConfig.getCertificateLocation();
+  }
+
+  /**
+   * Returns a X509 Certificate from the Certificate Holder.
+   *
+   * @param holder - Holder
+   * @return X509Certificate.
+   * @throws CertificateException - on Error.
+   */
+  public static X509Certificate getX509Certificate(X509CertificateHolder holder)
+      throws CertificateException {
+    return CERTIFICATE_CONVERTER.getCertificate(holder);
+  }
+
+  /**
+   * Get Certificate location.
+   *
+   * @return Path
+   */
+  public Path getLocation() {
+    return location;
+  }
+
+  /**
+   * Returns the Certificate as a PEM encoded String.
+   *
+   * @param x509CertHolder - X.509 Certificate Holder.
+   * @return PEM Encoded Certificate String.
+   * @throws SCMSecurityException - On failure to create a PEM String.
+   */
+  public String getPEMEncodedString(X509CertificateHolder x509CertHolder)
+      throws SCMSecurityException {
+    try {
+      StringWriter stringWriter = new StringWriter();
+      try (JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
+        pemWriter.writeObject(getX509Certificate(x509CertHolder));
+      }
+      return stringWriter.toString();
+    } catch (CertificateException | IOException e) {
+      LOG.error("Error in encoding certificate." + x509CertHolder
+          .getSubject().toString(), e);
+      throw new SCMSecurityException("PEM Encoding failed for certificate." +
+          x509CertHolder.getSubject().toString(), e);
+    }
+  }
+
+  /**
+   * Gets the X.509 Certificate from PEM encoded String.
+   *
+   * @param pemEncodedString - PEM encoded String.
+   * @return X509Certificate  - Certificate.
+   * @throws CertificateException - Thrown on Failure.
+   * @throws IOException          - Thrown on Failure.
+   */
+  public X509Certificate getX509Certificate(String pemEncodedString)
+      throws CertificateException, IOException {
+    CertificateFactory fact = CertificateFactory.getInstance("X.509");
+    try (InputStream input = IOUtils.toInputStream(pemEncodedString, UTF_8)) {
+      return (X509Certificate) fact.generateCertificate(input);
+    }
+  }
+
+  /**
+   * Write the Certificate pointed to the location by the configs.
+   *
+   * @param xCertificate - Certificate to write.
+   * @throws SCMSecurityException - on Error.
+   * @throws IOException - on Error.
+   */
+  public void writeCertificate(X509CertificateHolder xCertificate)
+      throws SCMSecurityException, IOException {
+    String pem = getPEMEncodedString(xCertificate);
+    writeCertificate(location.toAbsolutePath(),
+        this.securityConfig.getCertificateFileName(), pem, false);
+  }
+
+  /**
+   * Write the Certificate to the specific file.
+   *
+   * @param xCertificate - Certificate to write.
+   * @param fileName - file name to write to.
+   * @param overwrite - boolean value, true means overwrite an existing
+   * certificate.
+   * @throws SCMSecurityException - On Error.
+   * @throws IOException          - On Error.
+   */
+  public void writeCertificate(X509CertificateHolder xCertificate,
+      String fileName, boolean overwrite)
+      throws SCMSecurityException, IOException {
+    String pem = getPEMEncodedString(xCertificate);
+    writeCertificate(location.toAbsolutePath(), fileName, pem, overwrite);
+  }
+
+  /**
+   * Helper function that writes data to the file.
+   *
+   * @param basePath - Base Path where the file needs to written to.
+   * @param fileName - Certificate file name.
+   * @param pemEncodedCertificate - pemEncoded Certificate file.
+   * @param force - Overwrite if the file exists.
+   * @throws IOException - on Error.
+   */
+  public synchronized void writeCertificate(Path basePath, String fileName,
+      String pemEncodedCertificate, boolean force)
+      throws IOException {
+    File certificateFile =
+        Paths.get(basePath.toString(), fileName).toFile();
+    if (certificateFile.exists() && !force) {
+      throw new SCMSecurityException("Specified certificate file already " +
+          "exists.Please use force option if you want to overwrite it.");
+    }
+    if (!basePath.toFile().exists()) {
+      if (!basePath.toFile().mkdirs()) {
+        LOG.error("Unable to create file path. Path: {}", basePath);
+        throw new IOException("Creation of the directories failed."
+            + basePath.toString());
+      }
+    }
+    try (FileOutputStream file = new FileOutputStream(certificateFile)) {
+      IOUtils.write(pemEncodedCertificate, file, UTF_8);
+    }
+
+    Files.setPosixFilePermissions(certificateFile.toPath(), permissionSet);
+  }
+
+  /**
+   * Rertuns a default certificate using the default paths for this component.
+   *
+   * @return X509CertificateHolder.
+   * @throws SCMSecurityException - on Error.
+   * @throws CertificateException - on Error.
+   * @throws IOException          - on Error.
+   */
+  public X509CertificateHolder readCertificate() throws
+      CertificateException, IOException {
+    return readCertificate(this.location.toAbsolutePath(),
+        this.securityConfig.getCertificateFileName());
+  }
+
+  /**
+   * Returns the certificate from the specific PEM encoded file.
+   *
+   * @param basePath - base path
+   * @param fileName - fileName
+   * @return X%09 Certificate
+   * @throws IOException          - on Error.
+   * @throws SCMSecurityException - on Error.
+   * @throws CertificateException - on Error.
+   */
+  public synchronized X509CertificateHolder readCertificate(Path basePath,
+      String fileName) throws IOException, CertificateException {
+    File certificateFile = Paths.get(basePath.toString(), fileName).toFile();
+    return getX509CertificateHolder(certificateFile);
+  }
+
+  /**
+   * Helper function to read certificate.
+   *
+   * @param certificateFile - Full path to certificate file.
+   * @return X509CertificateHolder
+   * @throws IOException          - On Error.
+   * @throws CertificateException - On Error.
+   */
+  private X509CertificateHolder getX509CertificateHolder(File certificateFile)
+      throws IOException, CertificateException {
+    if (!certificateFile.exists()) {
+      throw new IOException("Unable to find the requested certificate. Path: "
+          + certificateFile.toString());
+    }
+    CertificateFactory fact = CertificateFactory.getInstance("X.509");
+    try (FileInputStream is = new FileInputStream(certificateFile)) {
+      return getCertificateHolder(
+          (X509Certificate) fact.generateCertificate(is));
+    }
+  }
+
+  /**
+   * Returns the Certificate holder from X509Ceritificate class.
+   *
+   * @param x509cert - Certificate class.
+   * @return X509CertificateHolder
+   * @throws CertificateEncodingException - on Error.
+   * @throws IOException                  - on Error.
+   */
+  public X509CertificateHolder getCertificateHolder(X509Certificate x509cert)
+      throws CertificateEncodingException, IOException {
+    return new X509CertificateHolder(x509cert.getEncoded());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
new file mode 100644
index 0000000..4971d4a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Certificate Utils.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
deleted file mode 100644
index 0e762a5..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509.certificates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
-import org.apache.logging.log4j.util.Strings;
-import org.bouncycastle.asn1.DEROctetString;
-import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.BasicConstraints;
-import org.bouncycastle.asn1.x509.Extension;
-import org.bouncycastle.asn1.x509.Extensions;
-import org.bouncycastle.asn1.x509.GeneralName;
-import org.bouncycastle.asn1.x509.GeneralNames;
-import org.bouncycastle.asn1.x509.KeyUsage;
-import org.bouncycastle.operator.ContentSigner;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-import org.bouncycastle.pkcs.PKCS10CertificationRequest;
-import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
-import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
-
-import java.io.IOException;
-import java.security.KeyPair;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-/**
- * A certificate sign request object that wraps operations to build a
- * PKCS10CertificationRequest to CA.
- */
-public final class CertificateSignRequest {
-  private final KeyPair keyPair;
-  private final SecurityConfig config;
-  private final Extensions extensions;
-  private String subject;
-  private String clusterID;
-  private String scmID;
-
-  /**
-   * Private Ctor for CSR.
-   *
-   * @param subject - Subject
-   * @param scmID - SCM ID
-   * @param clusterID - Cluster ID
-   * @param keyPair - KeyPair
-   * @param config - SCM Config
-   * @param extensions - CSR extensions
-   */
-  private CertificateSignRequest(String subject, String scmID, String clusterID,
-      KeyPair keyPair, SecurityConfig config, Extensions extensions) {
-    this.subject = subject;
-    this.clusterID = clusterID;
-    this.scmID = scmID;
-    this.keyPair = keyPair;
-    this.config = config;
-    this.extensions = extensions;
-  }
-
-  private PKCS10CertificationRequest generateCSR() throws
-      OperatorCreationException {
-    X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
-        clusterID);
-    PKCS10CertificationRequestBuilder p10Builder =
-        new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
-
-    ContentSigner contentSigner =
-        new JcaContentSignerBuilder(config.getSignatureAlgo())
-            .setProvider(config.getProvider())
-            .build(keyPair.getPrivate());
-
-    if (extensions != null) {
-      p10Builder.addAttribute(
-          PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
-    }
-    return p10Builder.build(contentSigner);
-  }
-
-  /**
-   * Builder class for Certificate Sign Request.
-   */
-  public static class Builder {
-    private String subject;
-    private String clusterID;
-    private String scmID;
-    private KeyPair key;
-    private SecurityConfig config;
-    private List<GeneralName> altNames;
-    private Boolean ca = false;
-
-    public CertificateSignRequest.Builder setConfiguration(
-        Configuration configuration) {
-      this.config = new SecurityConfig(configuration);
-      return this;
-    }
-
-    public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
-      this.key = keyPair;
-      return this;
-    }
-
-    public CertificateSignRequest.Builder setSubject(String subjectString) {
-      this.subject = subjectString;
-      return this;
-    }
-
-    public CertificateSignRequest.Builder setClusterID(String s) {
-      this.clusterID = s;
-      return this;
-    }
-
-    public CertificateSignRequest.Builder setScmID(String s) {
-      this.scmID = s;
-      return this;
-    }
-
-    // Support SAN extenion with DNS and RFC822 Name
-    // other name type will be added as needed.
-    public CertificateSignRequest.Builder addDnsName(String dnsName) {
-      Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
-      this.addAltName(GeneralName.dNSName, dnsName);
-      return this;
-    }
-
-    public CertificateSignRequest.Builder addRfc822Name(String name) {
-      Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
-      this.addAltName(GeneralName.rfc822Name, name);
-      return this;
-    }
-
-    // IP address is subject to change which is optional for now.
-    public CertificateSignRequest.Builder addIpAddress(String ip) {
-      Preconditions.checkNotNull(ip, "Ip address cannot be null");
-      this.addAltName(GeneralName.iPAddress, ip);
-      return this;
-    }
-
-    private CertificateSignRequest.Builder addAltName(int tag, String name) {
-      if (altNames == null) {
-        altNames = new ArrayList<>();
-      }
-      altNames.add(new GeneralName(tag, name));
-      return this;
-    }
-
-    public CertificateSignRequest.Builder setCA(Boolean isCA) {
-      this.ca = isCA;
-      return this;
-    }
-
-    private Extension getKeyUsageExtension() throws IOException {
-      int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
-          | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
-
-      if (ca) {
-        keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
-      }
-      KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
-      return new Extension(Extension.keyUsage, true,
-          new DEROctetString(keyUsage));
-    }
-
-    private Optional<Extension> getSubjectAltNameExtension() throws
-        IOException {
-      if (altNames != null) {
-        return Optional.of(new Extension(Extension.subjectAlternativeName,
-            true, new DEROctetString(new GeneralNames(
-            altNames.toArray(new GeneralName[altNames.size()])))));
-      }
-      return Optional.empty();
-    }
-
-    private Extension getBasicExtension() throws IOException {
-      // We don't set pathLenConstraint means no limit is imposed.
-      return new Extension(Extension.basicConstraints,
-          true, new DEROctetString(new BasicConstraints(ca)));
-    }
-
-    private Extensions createExtensions() throws IOException {
-      List<Extension> extensions = new ArrayList<>();
-
-      // Add basic extension
-      extensions.add(getBasicExtension());
-
-      // Add key usage extension
-      extensions.add(getKeyUsageExtension());
-
-      // Add subject alternate name extension
-      Optional<Extension> san = getSubjectAltNameExtension();
-      if (san.isPresent()) {
-        extensions.add(san.get());
-      }
-
-      return new Extensions(
-          extensions.toArray(new Extension[extensions.size()]));
-    }
-
-    public PKCS10CertificationRequest build() throws SCMSecurityException {
-      Preconditions.checkNotNull(key, "KeyPair cannot be null");
-      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
-          "cannot be blank");
-      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
-          "cannot be blank");
-      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
-          "be blank");
-
-      try {
-        CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
-            clusterID, key, config, createExtensions());
-        return csr.generateCSR();
-      } catch (IOException ioe) {
-        throw new CertificateException(String.format("Unable to create " +
-            "extension for certificate sign request for %s.", SecurityUtil
-            .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
-      } catch (OperatorCreationException ex) {
-        throw new CertificateException(String.format("Unable to create " +
-            "certificate sign request for %s.", SecurityUtil
-            .getDistinguishedName(subject, scmID, clusterID)),
-            ex.getCause());
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
deleted file mode 100644
index 85fba9b..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.certificates;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.util.Time;
-import org.apache.logging.log4j.util.Strings;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.BasicConstraints;
-import org.bouncycastle.asn1.x509.Extension;
-import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
-import org.bouncycastle.cert.CertIOException;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v3CertificateBuilder;
-import org.bouncycastle.operator.ContentSigner;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-
-import java.math.BigInteger;
-import java.security.KeyPair;
-import java.time.Duration;
-import java.util.Date;
-
-/**
- * A Self Signed Certificate with CA basic constraint can be used to boot-strap
- * a certificate infra-structure, if no external certificate is provided.
- */
-public final class SelfSignedCertificate {
-  private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
-  private String subject;
-  private String clusterID;
-  private String scmID;
-  private Date beginDate;
-  private Date endDate;
-  private KeyPair key;
-  private SecurityConfig config;
-  private boolean isCA;
-
-  /**
-   * Private Ctor invoked only via Builder Interface.
-   * @param subject - Subject
-   * @param scmID - SCM ID
-   * @param clusterID - Cluster ID
-   * @param beginDate - NotBefore
-   * @param endDate - Not After
-   * @param configuration - SCM Config
-   * @param keyPair - KeyPair
-   * @param ca - isCA?
-   */
-  private SelfSignedCertificate(String subject, String scmID, String clusterID,
-      Date beginDate, Date endDate, SecurityConfig configuration,
-      KeyPair keyPair, boolean ca) {
-    this.subject = subject;
-    this.clusterID = clusterID;
-    this.scmID = scmID;
-    this.beginDate = beginDate;
-    this.endDate = endDate;
-    config = configuration;
-    this.key = keyPair;
-    this.isCA = ca;
-  }
-
-  @VisibleForTesting
-  public static String getNameFormat() {
-    return NAME_FORMAT;
-  }
-
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  private X509CertificateHolder generateCertificate()
-      throws OperatorCreationException, CertIOException {
-    // For the Root Certificate we form the name from Subject, SCM ID and
-    // Cluster ID.
-    String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
-    X500Name name = new X500Name(dnName);
-    byte[] encoded = key.getPublic().getEncoded();
-    SubjectPublicKeyInfo publicKeyInfo =
-        SubjectPublicKeyInfo.getInstance(encoded);
-
-
-    ContentSigner contentSigner =
-        new JcaContentSignerBuilder(config.getSignatureAlgo())
-            .setProvider(config.getProvider()).build(key.getPrivate());
-
-    // Please note: Since this is a root certificate we use "ONE" as the
-    // serial number. Also note that skip enforcing locale or UTC. We are
-    // trying to operate at the Days level, hence Time zone is also skipped for
-    // now.
-    BigInteger serial = BigInteger.ONE;
-    if (!isCA) {
-      serial = new BigInteger(Long.toString(Time.monotonicNow()));
-    }
-
-    X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
-        serial, beginDate, endDate, name, publicKeyInfo);
-
-    if (isCA) {
-      builder.addExtension(Extension.basicConstraints, true,
-          new BasicConstraints(true));
-    }
-    return builder.build(contentSigner);
-  }
-
-  /**
-   * Builder class for Root Certificates.
-   */
-  public static class Builder {
-    private String subject;
-    private String clusterID;
-    private String scmID;
-    private Date beginDate;
-    private Date endDate;
-    private KeyPair key;
-    private SecurityConfig config;
-    private boolean isCA;
-
-    public Builder setConfiguration(Configuration configuration) {
-      this.config = new SecurityConfig(configuration);
-      return this;
-    }
-
-    public Builder setKey(KeyPair keyPair) {
-      this.key = keyPair;
-      return this;
-    }
-
-    public Builder setSubject(String subjectString) {
-      this.subject = subjectString;
-      return this;
-    }
-
-    public Builder setClusterID(String s) {
-      this.clusterID = s;
-      return this;
-    }
-
-    public Builder setScmID(String s) {
-      this.scmID = s;
-      return this;
-    }
-
-    public Builder setBeginDate(Date date) {
-      this.beginDate = new Date(date.toInstant().toEpochMilli());
-      return this;
-    }
-
-    public Builder setEndDate(Date date) {
-      this.endDate = new Date(date.toInstant().toEpochMilli());
-      return this;
-    }
-
-    public Builder makeCA() {
-      isCA = true;
-      return this;
-    }
-
-    public X509CertificateHolder build() throws SCMSecurityException {
-      Preconditions.checkNotNull(key, "Key cannot be null");
-      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
-          "cannot be blank");
-      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
-          "cannot be blank");
-      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
-          "be blank");
-
-      Preconditions.checkArgument(beginDate.before(endDate), "Certificate " +
-          "begin date should be before end date");
-
-      Duration certDuration = Duration.between(beginDate.toInstant(),
-          endDate.toInstant());
-      Preconditions.checkArgument(
-          certDuration.compareTo(config.getMaxCertificateDuration()) < 0,
-          "Certificate life time cannot be greater than max configured value.");
-
-
-      SelfSignedCertificate rootCertificate =
-          new SelfSignedCertificate(this.subject,
-          this.scmID, this.clusterID, this.beginDate, this.endDate,
-          this.config, key, isCA);
-      try {
-        return rootCertificate.generateCertificate();
-      } catch (OperatorCreationException | CertIOException e) {
-        throw new CertificateException("Unable to create root certificate.",
-            e.getCause());
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
deleted file mode 100644
index e88737c..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Utils for Certificates.
- */
-package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
new file mode 100644
index 0000000..3624b32
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CertificateServer.
+ */
+public final class CertificateSignRequest {
+  private final KeyPair keyPair;
+  private final SecurityConfig config;
+  private final Extensions extensions;
+  private String subject;
+  private String clusterID;
+  private String scmID;
+
+  /**
+   * Private Ctor for CSR.
+   *
+   * @param subject - Subject
+   * @param scmID - SCM ID
+   * @param clusterID - Cluster ID
+   * @param keyPair - KeyPair
+   * @param config - SCM Config
+   * @param extensions - CSR extensions
+   */
+  private CertificateSignRequest(String subject, String scmID, String clusterID,
+      KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+    this.keyPair = keyPair;
+    this.config = config;
+    this.extensions = extensions;
+  }
+
+  private PKCS10CertificationRequest generateCSR() throws
+      OperatorCreationException {
+    X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+        clusterID);
+    PKCS10CertificationRequestBuilder p10Builder =
+        new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+    ContentSigner contentSigner =
+        new JcaContentSignerBuilder(config.getSignatureAlgo())
+            .setProvider(config.getProvider())
+            .build(keyPair.getPrivate());
+
+    if (extensions != null) {
+      p10Builder.addAttribute(
+          PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+    }
+    return p10Builder.build(contentSigner);
+  }
+
+  /**
+   * Builder class for Certificate Sign Request.
+   */
+  public static class Builder {
+    private String subject;
+    private String clusterID;
+    private String scmID;
+    private KeyPair key;
+    private SecurityConfig config;
+    private List<GeneralName> altNames;
+    private Boolean ca = false;
+
+    public CertificateSignRequest.Builder setConfiguration(
+        Configuration configuration) {
+      this.config = new SecurityConfig(configuration);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+      this.key = keyPair;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setSubject(String subjectString) {
+      this.subject = subjectString;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setClusterID(String s) {
+      this.clusterID = s;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setScmID(String s) {
+      this.scmID = s;
+      return this;
+    }
+
+    // Support SAN extenion with DNS and RFC822 Name
+    // other name type will be added as needed.
+    public CertificateSignRequest.Builder addDnsName(String dnsName) {
+      Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+      this.addAltName(GeneralName.dNSName, dnsName);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder addRfc822Name(String name) {
+      Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+      this.addAltName(GeneralName.rfc822Name, name);
+      return this;
+    }
+
+    // IP address is subject to change which is optional for now.
+    public CertificateSignRequest.Builder addIpAddress(String ip) {
+      Preconditions.checkNotNull(ip, "Ip address cannot be null");
+      this.addAltName(GeneralName.iPAddress, ip);
+      return this;
+    }
+
+    private CertificateSignRequest.Builder addAltName(int tag, String name) {
+      if (altNames == null) {
+        altNames = new ArrayList<>();
+      }
+      altNames.add(new GeneralName(tag, name));
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setCA(Boolean isCA) {
+      this.ca = isCA;
+      return this;
+    }
+
+    private Extension getKeyUsageExtension() throws IOException {
+      int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+          | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+      if (ca) {
+        keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+      }
+      KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+      return new Extension(Extension.keyUsage, true,
+          new DEROctetString(keyUsage));
+    }
+
+    private Optional<Extension> getSubjectAltNameExtension() throws
+        IOException {
+      if (altNames != null) {
+        return Optional.of(new Extension(Extension.subjectAlternativeName,
+            true, new DEROctetString(new GeneralNames(
+            altNames.toArray(new GeneralName[altNames.size()])))));
+      }
+      return Optional.empty();
+    }
+
+    private Extension getBasicExtension() throws IOException {
+      // We don't set pathLenConstraint means no limit is imposed.
+      return new Extension(Extension.basicConstraints,
+          true, new DEROctetString(new BasicConstraints(ca)));
+    }
+
+    private Extensions createExtensions() throws IOException {
+      List<Extension> extensions = new ArrayList<>();
+
+      // Add basic extension
+      extensions.add(getBasicExtension());
+
+      // Add key usage extension
+      extensions.add(getKeyUsageExtension());
+
+      // Add subject alternate name extension
+      Optional<Extension> san = getSubjectAltNameExtension();
+      if (san.isPresent()) {
+        extensions.add(san.get());
+      }
+
+      return new Extensions(
+          extensions.toArray(new Extension[extensions.size()]));
+    }
+
+    public PKCS10CertificationRequest build() throws SCMSecurityException {
+      Preconditions.checkNotNull(key, "KeyPair cannot be null");
+      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+          "be blank");
+
+      try {
+        CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+            clusterID, key, config, createExtensions());
+        return csr.generateCSR();
+      } catch (IOException ioe) {
+        throw new CertificateException(String.format("Unable to create " +
+            "extension for certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+      } catch (OperatorCreationException ex) {
+        throw new CertificateException(String.format("Unable to create " +
+            "certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)),
+            ex.getCause());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
new file mode 100644
index 0000000..1fd6d7c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.util.Time;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+
+/**
+ * A Self Signed Certificate with CertificateServer basic constraint can be used
+ * to bootstrap a certificate infrastructure, if no external certificate is
+ * provided.
+ */
+public final class SelfSignedCertificate {
+  private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+  private String subject;
+  private String clusterID;
+  private String scmID;
+  private LocalDate beginDate;
+  private LocalDate endDate;
+  private KeyPair key;
+  private SecurityConfig config;
+
+  /**
+   * Private Ctor invoked only via Builder Interface.
+   *
+   * @param subject - Subject
+   * @param scmID - SCM ID
+   * @param clusterID - Cluster ID
+   * @param beginDate - NotBefore
+   * @param endDate - Not After
+   * @param configuration - SCM Config
+   * @param keyPair - KeyPair
+   */
+  private SelfSignedCertificate(String subject, String scmID, String clusterID,
+      LocalDate beginDate, LocalDate endDate, SecurityConfig configuration,
+      KeyPair keyPair) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+    this.beginDate = beginDate;
+    this.endDate = endDate;
+    config = configuration;
+    this.key = keyPair;
+  }
+
+  @VisibleForTesting
+  public static String getNameFormat() {
+    return NAME_FORMAT;
+  }
+
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  private X509CertificateHolder generateCertificate(boolean isCA)
+      throws OperatorCreationException, IOException {
+    // For the Root Certificate we form the name from Subject, SCM ID and
+    // Cluster ID.
+    String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
+    X500Name name = new X500Name(dnName);
+    byte[] encoded = key.getPublic().getEncoded();
+    SubjectPublicKeyInfo publicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(encoded);
+
+
+    ContentSigner contentSigner =
+        new JcaContentSignerBuilder(config.getSignatureAlgo())
+            .setProvider(config.getProvider()).build(key.getPrivate());
+
+    // Please note: Since this is a root certificate we use "ONE" as the
+    // serial number. Also note that skip enforcing locale or UTC. We are
+    // trying to operate at the Days level, hence Time zone is also skipped for
+    // now.
+    BigInteger serial = BigInteger.ONE;
+    if (!isCA) {
+      serial = new BigInteger(Long.toString(Time.monotonicNow()));
+    }
+
+    ZoneOffset zoneOffset =
+        beginDate.atStartOfDay(ZoneOffset.systemDefault()).getOffset();
+
+    // Valid from the Start of the day when we generate this Certificate.
+    Date validFrom =
+        Date.from(beginDate.atTime(LocalTime.MIN).toInstant(zoneOffset));
+
+    // Valid till end day finishes.
+    Date validTill =
+        Date.from(endDate.atTime(LocalTime.MAX).toInstant(zoneOffset));
+
+    X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
+        serial, validFrom, validTill, name, publicKeyInfo);
+
+    if (isCA) {
+      builder.addExtension(Extension.basicConstraints, true,
+          new BasicConstraints(true));
+      int keyUsageFlag = KeyUsage.keyCertSign | KeyUsage.cRLSign;
+      KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+      builder.addExtension(Extension.keyUsage, false,
+          new DEROctetString(keyUsage));
+    }
+    return builder.build(contentSigner);
+  }
+
+  /**
+   * Builder class for Root Certificates.
+   */
+  public static class Builder {
+    private String subject;
+    private String clusterID;
+    private String scmID;
+    private LocalDate beginDate;
+    private LocalDate endDate;
+    private KeyPair key;
+    private SecurityConfig config;
+    private boolean isCA;
+
+    public Builder setConfiguration(Configuration configuration) {
+      this.config = new SecurityConfig(configuration);
+      return this;
+    }
+
+    public Builder setKey(KeyPair keyPair) {
+      this.key = keyPair;
+      return this;
+    }
+
+    public Builder setSubject(String subjectString) {
+      this.subject = subjectString;
+      return this;
+    }
+
+    public Builder setClusterID(String s) {
+      this.clusterID = s;
+      return this;
+    }
+
+    public Builder setScmID(String s) {
+      this.scmID = s;
+      return this;
+    }
+
+    public Builder setBeginDate(LocalDate date) {
+      this.beginDate = date;
+      return this;
+    }
+
+    public Builder setEndDate(LocalDate date) {
+      this.endDate = date;
+      return this;
+    }
+
+    public Builder makeCA() {
+      isCA = true;
+      return this;
+    }
+
+    public X509CertificateHolder build()
+        throws SCMSecurityException, IOException {
+      Preconditions.checkNotNull(key, "Key cannot be null");
+      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+          "be blank");
+
+      Preconditions.checkArgument(beginDate.isBefore(endDate), "Certificate " +
+          "begin date should be before end date");
+
+      // We just read the beginDate and EndDate as Start of the Day and
+      // confirm that we do not violate the maxDuration Config.
+      Duration certDuration = Duration.between(beginDate.atStartOfDay(),
+          endDate.atStartOfDay());
+      Duration maxDuration = config.getMaxCertificateDuration();
+      if (certDuration.compareTo(maxDuration) > 0) {
+        throw new SCMSecurityException("The cert duration violates the " +
+            "maximum configured value. Please check the hdds.x509.max" +
+            ".duration config key. Current Value: " + certDuration +
+            " config: " + maxDuration);
+      }
+
+      SelfSignedCertificate rootCertificate =
+          new SelfSignedCertificate(this.subject,
+              this.scmID, this.clusterID, this.beginDate, this.endDate,
+              this.config, key);
+      try {
+        return rootCertificate.generateCertificate(isCA);
+      } catch (OperatorCreationException | CertIOException e) {
+        throw new CertificateException("Unable to create root certificate.",
+            e.getCause());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
new file mode 100644
index 0000000..945adc9
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Utils for Certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[02/50] [abbrv] hadoop git commit: YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda)

Posted by xy...@apache.org.
YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda)

Change-Id: I4eb58e9d251d5f54e7feffc4fbb813b4f5ae4b1b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ebeda98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ebeda98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ebeda98

Branch: refs/heads/HDDS-4
Commit: 8ebeda98a9d3eac45598a33bae3d62e3ebb92cad
Parents: 9ed8756
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:36:30 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:36:30 2018 -0800

----------------------------------------------------------------------
 .../linux/resources/gpu/TestGpuResourceHandler.java                | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ebeda98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 10e5cd1..18785e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -465,7 +465,7 @@ public class TestGpuResourceHandler {
       caughtException = true;
     }
     Assert.assertTrue(
-        "Should fail since requested device Id is not in allowed list",
+        "Should fail since requested device Id is already assigned",
         caughtException);
 
     // Make sure internal state not changed.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[12/50] [abbrv] hadoop git commit: HDDS-850. ReadStateMachineData hits OverlappingFileLockException in ContainerStateMachine. Contributed by Shashikant Banerjee.

Posted by xy...@apache.org.
HDDS-850. ReadStateMachineData hits OverlappingFileLockException in ContainerStateMachine. Contributed by Shashikant Banerjee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e102f9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e102f9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e102f9a

Branch: refs/heads/HDDS-4
Commit: 5e102f9aa54d3057ef5f0755d45428f22a24990b
Parents: 7eb0d3a
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Thu Nov 29 22:20:08 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Thu Nov 29 22:20:08 2018 +0530

----------------------------------------------------------------------
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |   8 ++
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |   9 ++
 .../main/proto/DatanodeContainerProtocol.proto  |   1 +
 .../common/src/main/resources/ozone-default.xml |   8 ++
 .../server/ratis/ContainerStateMachine.java     | 134 +++++++++++--------
 .../server/ratis/XceiverServerRatis.java        |  14 +-
 .../container/keyvalue/KeyValueHandler.java     |   7 +-
 .../keyvalue/impl/ChunkManagerImpl.java         |  11 +-
 .../keyvalue/interfaces/ChunkManager.java       |   5 +-
 .../keyvalue/TestChunkManagerImpl.java          |   6 +-
 .../common/impl/TestContainerPersistence.java   |  11 +-
 11 files changed, 143 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 6733b8e..062b101 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -93,6 +93,14 @@ public final class ScmConfigKeys {
   public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE =
       "dfs.container.ratis.log.queue.size";
   public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT = 128;
+
+  // expiry interval stateMachineData cache entry inside containerStateMachine
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+      "dfs.container.ratis.statemachine.cache.expiry.interval";
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+      "10s";
   public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
       "dfs.ratis.client.request.timeout.duration";
   public static final TimeDuration

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 879f773..df233f7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -249,6 +249,15 @@ public final class OzoneConfigKeys {
       DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
       ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
 
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+      ScmConfigKeys.
+          DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL;
+  public static final String
+      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+      ScmConfigKeys.
+          DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT;
+
   public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
       "dfs.container.ratis.datanode.storage.dir";
   public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 3695b6b..5237af8 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -392,6 +392,7 @@ message  WriteChunkResponseProto {
 message  ReadChunkRequestProto  {
   required DatanodeBlockID blockID = 1;
   required ChunkInfo chunkData = 2;
+  optional bool readFromTmpFile = 3 [default = false];
 }
 
 message  ReadChunkResponseProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aa22b2b..0545805 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -190,6 +190,14 @@
     </description>
   </property>
   <property>
+    <name>dfs.container.ratis.statemachine.cache.expiry.interval</name>
+    <value>10s</value>
+    <tag>OZONE, RATIS, PERFORMANCE</tag>
+    <description>The interval till which the stateMachine data in ratis
+      will be cached inside the ContainerStateMachine.
+    </description>
+  </property>
+  <property>
     <name>dfs.ratis.client.request.timeout.duration</name>
     <value>3s</value>
     <tag>OZONE, RATIS, MANAGEMENT</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 24ecbb2..2e0c70e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -19,6 +19,8 @@
 package org.apache.hadoop.ozone.container.common.transport.server.ratis;
 
 import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -27,6 +29,7 @@ import org.apache.ratis.protocol.RaftGroup;
 import org.apache.ratis.protocol.RaftGroupId;
 import org.apache.ratis.server.RaftServer;
 import org.apache.ratis.server.impl.RaftServerConstants;
+import org.apache.ratis.server.impl.RaftServerProxy;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
 import org.apache.ratis.thirdparty.com.google.protobuf
@@ -60,12 +63,16 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
 /** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
@@ -116,12 +123,11 @@ public class ContainerStateMachine extends BaseStateMachine {
   private final XceiverServerRatis ratisServer;
   private final ConcurrentHashMap<Long, CompletableFuture<Message>>
       writeChunkFutureMap;
-  private final ConcurrentHashMap<Long, CompletableFuture<Message>>
-      createContainerFutureMap;
   private ExecutorService[] executors;
   private final int numExecutors;
   private final Map<Long, Long> applyTransactionCompletionMap;
   private long lastIndex;
+  private final Cache<Long, ByteString> stateMachineDataCache;
   /**
    * CSM metrics.
    */
@@ -129,7 +135,7 @@ public class ContainerStateMachine extends BaseStateMachine {
 
   public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher,
       ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer,
-      List<ExecutorService> executors) {
+      List<ExecutorService> executors, long expiryInterval) {
     this.gid = gid;
     this.dispatcher = dispatcher;
     this.chunkExecutor = chunkExecutor;
@@ -138,9 +144,13 @@ public class ContainerStateMachine extends BaseStateMachine {
     this.numExecutors = executors.size();
     this.executors = executors.toArray(new ExecutorService[numExecutors]);
     this.writeChunkFutureMap = new ConcurrentHashMap<>();
-    this.createContainerFutureMap = new ConcurrentHashMap<>();
     applyTransactionCompletionMap = new ConcurrentHashMap<>();
     this.lastIndex = RaftServerConstants.INVALID_LOG_INDEX;
+    stateMachineDataCache = CacheBuilder.newBuilder()
+        .expireAfterAccess(expiryInterval, TimeUnit.MILLISECONDS)
+        // set the limit on no of cached entries equal to no of max threads
+        // executing writeStateMachineData
+        .maximumSize(chunkExecutor.getCorePoolSize()).build();
   }
 
   @Override
@@ -257,14 +267,6 @@ public class ContainerStateMachine extends BaseStateMachine {
           .setStateMachineData(dataContainerCommandProto.toByteString())
           .setLogData(commitContainerCommandProto.toByteString())
           .build();
-    } else if (proto.getCmdType() == Type.CreateContainer) {
-      return TransactionContext.newBuilder()
-          .setClientRequest(request)
-          .setStateMachine(this)
-          .setServerRole(RaftPeerRole.LEADER)
-          .setStateMachineData(request.getMessage().getContent())
-          .setLogData(request.getMessage().getContent())
-          .build();
     } else {
       return TransactionContext.newBuilder()
           .setClientRequest(request)
@@ -310,17 +312,17 @@ public class ContainerStateMachine extends BaseStateMachine {
   private CompletableFuture<Message> handleWriteChunk(
       ContainerCommandRequestProto requestProto, long entryIndex) {
     final WriteChunkRequestProto write = requestProto.getWriteChunk();
-    long containerID = write.getBlockID().getContainerID();
-    CompletableFuture<Message> future =
-        createContainerFutureMap.get(containerID);
-    CompletableFuture<Message> writeChunkFuture;
-    if (future != null) {
-      writeChunkFuture = future.thenApplyAsync(
-          v -> runCommand(requestProto), chunkExecutor);
-    } else {
-      writeChunkFuture = CompletableFuture.supplyAsync(
-          () -> runCommand(requestProto), chunkExecutor);
+    RaftServer server = ratisServer.getServer();
+    Preconditions.checkState(server instanceof RaftServerProxy);
+    try {
+      if (((RaftServerProxy) server).getImpl(gid).isLeader()) {
+        stateMachineDataCache.put(entryIndex, write.getData());
+      }
+    } catch (IOException ioe) {
+      return completeExceptionally(ioe);
     }
+    CompletableFuture<Message> writeChunkFuture = CompletableFuture
+        .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
     writeChunkFutureMap.put(entryIndex, writeChunkFuture);
     LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
         + " logIndex " + entryIndex + " chunkName " + write.getChunkData()
@@ -337,14 +339,6 @@ public class ContainerStateMachine extends BaseStateMachine {
     return writeChunkFuture;
   }
 
-  private CompletableFuture<Message> handleCreateContainer(
-      ContainerCommandRequestProto requestProto) {
-    long containerID = requestProto.getContainerID();
-    createContainerFutureMap.
-        computeIfAbsent(containerID, k -> new CompletableFuture<>());
-    return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
-  }
-
   /*
    * writeStateMachineData calls are not synchronized with each other
    * and also with applyTransaction.
@@ -356,9 +350,10 @@ public class ContainerStateMachine extends BaseStateMachine {
       final ContainerCommandRequestProto requestProto =
           getRequestProto(getStateMachineData(entry.getStateMachineLogEntry()));
       Type cmdType = requestProto.getCmdType();
+
+      // For only writeChunk, there will be writeStateMachineData call.
+      // CreateContainer will happen as a part of writeChunk only.
       switch (cmdType) {
-      case CreateContainer:
-        return handleCreateContainer(requestProto);
       case WriteChunk:
         return handleWriteChunk(requestProto, entry.getIndex());
       default:
@@ -397,7 +392,10 @@ public class ContainerStateMachine extends BaseStateMachine {
     ReadChunkRequestProto.Builder readChunkRequestProto =
         ReadChunkRequestProto.newBuilder()
             .setBlockID(writeChunkRequestProto.getBlockID())
-            .setChunkData(writeChunkRequestProto.getChunkData());
+            .setChunkData(writeChunkRequestProto.getChunkData())
+            // set readFromTempFile to true in case, the chunkFile does
+            // not exist as applyTransaction is not executed for this entry yet.
+            .setReadFromTmpFile(true);
     ContainerCommandRequestProto dataContainerCommandProto =
         ContainerCommandRequestProto.newBuilder(requestProto)
             .setCmdType(Type.ReadChunk)
@@ -409,15 +407,39 @@ public class ContainerStateMachine extends BaseStateMachine {
         dispatchCommand(dataContainerCommandProto);
     ReadChunkResponseProto responseProto = response.getReadChunk();
 
+    ByteString data = responseProto.getData();
     // assert that the response has data in it.
-    Preconditions.checkNotNull(responseProto.getData());
+    Preconditions.checkNotNull(data);
+    return data;
+  }
 
+  /**
+   * Reads the Entry from the Cache or loads it back by reading from disk.
+   */
+  private ByteString getCachedStateMachineData(Long logIndex,
+      ContainerCommandRequestProto requestProto) throws ExecutionException {
+    try {
+      return reconstructWriteChunkRequest(
+          stateMachineDataCache.get(logIndex, new Callable<ByteString>() {
+            @Override
+            public ByteString call() throws Exception {
+              return readStateMachineData(requestProto);
+            }
+          }), requestProto);
+    } catch (ExecutionException e) {
+      throw e;
+    }
+  }
+
+  private ByteString reconstructWriteChunkRequest(ByteString data,
+      ContainerCommandRequestProto requestProto) {
+    WriteChunkRequestProto writeChunkRequestProto =
+        requestProto.getWriteChunk();
     // reconstruct the write chunk request
     final WriteChunkRequestProto.Builder dataWriteChunkProto =
         WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
             // adding the state machine data
-            .setData(responseProto.getData())
-            .setStage(Stage.WRITE_DATA);
+            .setData(data).setStage(Stage.WRITE_DATA);
 
     ContainerCommandRequestProto.Builder newStateMachineProto =
         ContainerCommandRequestProto.newBuilder(requestProto)
@@ -455,18 +477,21 @@ public class ContainerStateMachine extends BaseStateMachine {
     if (!getStateMachineData(smLogEntryProto).isEmpty()) {
       return CompletableFuture.completedFuture(ByteString.EMPTY);
     }
-
     try {
       final ContainerCommandRequestProto requestProto =
           getRequestProto(entry.getStateMachineLogEntry().getLogData());
       // readStateMachineData should only be called for "write" to Ratis.
       Preconditions.checkArgument(!HddsUtils.isReadOnly(requestProto));
-
       if (requestProto.getCmdType() == Type.WriteChunk) {
-        return CompletableFuture.supplyAsync(() ->
-                readStateMachineData(requestProto), chunkExecutor);
-      } else if (requestProto.getCmdType() == Type.CreateContainer) {
-        return CompletableFuture.completedFuture(requestProto.toByteString());
+        CompletableFuture<ByteString> future = new CompletableFuture<>();
+        return future.supplyAsync(() -> {
+          try {
+            return getCachedStateMachineData(entry.getIndex(), requestProto);
+          } catch (ExecutionException e) {
+            future.completeExceptionally(e);
+            return null;
+          }
+        }, chunkExecutor);
       } else {
         throw new IllegalStateException("Cmd type:" + requestProto.getCmdType()
             + " cannot have state machine data");
@@ -559,19 +584,6 @@ public class ContainerStateMachine extends BaseStateMachine {
         future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
             getCommandExecutor(requestProto));
       }
-      // Mark the createContainerFuture complete so that writeStateMachineData
-      // for WriteChunk gets unblocked
-      if (cmdType == Type.CreateContainer) {
-        long containerID = requestProto.getContainerID();
-        future.thenApply(
-            r -> {
-              createContainerFutureMap.remove(containerID).complete(null);
-              LOG.info("create Container Transaction completed for container " +
-                  containerID + " log index " + index);
-              return r;
-            });
-      }
-
       lastIndex = index;
       future.thenAccept(m -> {
         final Long previous =
@@ -593,6 +605,11 @@ public class ContainerStateMachine extends BaseStateMachine {
     return future;
   }
 
+  private void evictStateMachineCache() {
+    stateMachineDataCache.invalidateAll();
+    stateMachineDataCache.cleanUp();
+  }
+
   @Override
   public void notifySlowness(RaftGroup group, RoleInfoProto roleInfoProto) {
     ratisServer.handleNodeSlowness(group, roleInfoProto);
@@ -605,6 +622,13 @@ public class ContainerStateMachine extends BaseStateMachine {
   }
 
   @Override
+  public void notifyNotLeader(Collection<TransactionContext> pendingEntries)
+      throws IOException {
+    evictStateMachineCache();
+  }
+
+  @Override
   public void close() throws IOException {
+    evictStateMachineCache();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a4927e0..2675483 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -105,6 +105,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
   private final StateContext context;
   private final ReplicationLevel replicationLevel;
   private long nodeFailureTimeoutMs;
+  private final long cacheEntryExpiryInteval;
+
 
   private XceiverServerRatis(DatanodeDetails dd, int port,
       ContainerDispatcher dispatcher, Configuration conf, StateContext context)
@@ -128,6 +130,11 @@ public final class XceiverServerRatis implements XceiverServerSpi {
         conf.getEnum(OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY,
             OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT);
     this.executors = new ArrayList<>();
+    cacheEntryExpiryInteval = conf.getTimeDuration(OzoneConfigKeys.
+            DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL,
+        OzoneConfigKeys.
+            DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT,
+        TimeUnit.MILLISECONDS);
     this.dispatcher = dispatcher;
     for (int i = 0; i < numContainerOpExecutors; i++) {
       executors.add(Executors.newSingleThreadExecutor());
@@ -141,8 +148,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
   }
 
   private ContainerStateMachine getStateMachine(RaftGroupId gid) {
-    return new ContainerStateMachine(gid, dispatcher, chunkExecutor,
-            this, Collections.unmodifiableList(executors));
+    return new ContainerStateMachine(gid, dispatcher, chunkExecutor, this,
+        Collections.unmodifiableList(executors), cacheEntryExpiryInteval);
   }
 
   private RaftProperties newRaftProperties(Configuration conf) {
@@ -304,6 +311,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
     RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties,
         numSyncRetries);
 
+    // Enable the StateMachineCaching
+    RaftServerConfigKeys.Log.StateMachineData
+        .setCachingEnabled(properties, true);
     return properties;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 36c48c5..a16129e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -531,7 +531,8 @@ public class KeyValueHandler extends Handler {
           .getChunkData());
       Preconditions.checkNotNull(chunkInfo);
 
-      data = chunkManager.readChunk(kvContainer, blockID, chunkInfo);
+      data = chunkManager.readChunk(kvContainer, blockID, chunkInfo,
+          request.getReadChunk().getReadFromTmpFile());
       metrics.incContainerBytesStats(Type.ReadChunk, data.length);
     } catch (StorageContainerException ex) {
       return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -702,8 +703,10 @@ public class KeyValueHandler extends Handler {
       ContainerProtos.ChunkInfo chunkInfo = null;
       ByteString dataBuf = ByteString.EMPTY;
       for (ContainerProtos.ChunkInfo chunk : responseData.getChunks()) {
+        // if the block is committed, all chunks must have been committed.
+        // Tmp chunk files won't exist here.
         byte[] data = chunkManager.readChunk(kvContainer, blockID,
-            ChunkInfo.getFromProtoBuf(chunk));
+            ChunkInfo.getFromProtoBuf(chunk), false);
         ByteString current = ByteString.copyFrom(data);
         dataBuf = dataBuf.concat(current);
         chunkInfo = chunk;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index a1c3d01..e41346c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -173,13 +173,14 @@ public class ChunkManagerImpl implements ChunkManager {
    * @param container - Container for the chunk
    * @param blockID - ID of the block.
    * @param info - ChunkInfo.
+   * @param readFromTmpFile whether to read from tmp chunk file or not.
    * @return byte array
    * @throws StorageContainerException
    * TODO: Right now we do not support partial reads and writes of chunks.
    * TODO: Explore if we need to do that for ozone.
    */
-  public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info)
-      throws StorageContainerException {
+  public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+      boolean readFromTmpFile) throws StorageContainerException {
     try {
       KeyValueContainerData containerData = (KeyValueContainerData) container
           .getContainerData();
@@ -194,6 +195,12 @@ public class ChunkManagerImpl implements ChunkManager {
       if (containerData.getLayOutVersion() == ChunkLayOutVersion
           .getLatestVersion().getVersion()) {
         File chunkFile = ChunkUtils.getChunkFile(containerData, info);
+
+        // In case the chunk file does not exist but tmp chunk file exist,
+        // read from tmp chunk file if readFromTmpFile is set to true
+        if (!chunkFile.exists() && readFromTmpFile) {
+          chunkFile = getTmpChunkFile(chunkFile, info);
+        }
         data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
         containerData.incrReadCount();
         long length = chunkFile.length();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 3b06585..f90346e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -51,14 +51,15 @@ public interface ChunkManager {
    * @param container - Container for the chunk
    * @param blockID - ID of the block.
    * @param info - ChunkInfo.
+   * @param readFromTmpFile whether to read from tmp chunk file or not
    * @return  byte array
    * @throws StorageContainerException
    *
    * TODO: Right now we do not support partial reads and writes of chunks.
    * TODO: Explore if we need to do that for ozone.
    */
-  byte[] readChunk(Container container, BlockID blockID, ChunkInfo info) throws
-      StorageContainerException;
+  byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+      boolean readFromTmpFile) throws StorageContainerException;
 
   /**
    * Deletes a given chunk.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 717dd05..ef77204 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -189,7 +189,7 @@ public class TestChunkManagerImpl {
     checkWriteIOStats(data.length, 1);
     checkReadIOStats(0, 0);
     byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
-        chunkInfo);
+        chunkInfo, false);
     assertEquals(expectedData.length, data.length);
     assertTrue(Arrays.equals(expectedData, data));
     checkReadIOStats(data.length, 1);
@@ -226,7 +226,7 @@ public class TestChunkManagerImpl {
     try {
       // trying to read a chunk, where chunk file does not exist
       byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
-          chunkInfo);
+          chunkInfo, false);
       fail("testReadChunkFileNotExists failed");
     } catch (StorageContainerException ex) {
       GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
@@ -249,7 +249,7 @@ public class TestChunkManagerImpl {
     for (int i=0; i<100; i++) {
       chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
           .getLocalID(), i), 0, data.length);
-      chunkManager.readChunk(keyValueContainer, blockID, chunkInfo);
+      chunkManager.readChunk(keyValueContainer, blockID, chunkInfo, false);
     }
     checkReadIOStats(data.length*100, 100);
     assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index fb7b0c5..5d19a10 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -406,7 +406,7 @@ public class TestContainerPersistence {
       for (int x = 0; x < chunkCount; x++) {
         String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
         ChunkInfo info = fileHashMap.get(fileName);
-        byte[] data = chunkManager.readChunk(container, blockID, info);
+        byte[] data = chunkManager.readChunk(container, blockID, info, false);
         ChecksumData checksumData = checksum.computeChecksum(data);
         Assert.assertEquals(info.getChecksumData(), checksumData);
       }
@@ -435,11 +435,11 @@ public class TestContainerPersistence {
     chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
         COMBINED);
 
-    byte[] readData = chunkManager.readChunk(container, blockID, info);
+    byte[] readData = chunkManager.readChunk(container, blockID, info, false);
     assertTrue(Arrays.equals(data, readData));
 
     ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length);
-    byte[] readData2 = chunkManager.readChunk(container, blockID, info2);
+    byte[] readData2 = chunkManager.readChunk(container, blockID, info2, false);
     assertEquals(length, readData2.length);
     assertTrue(Arrays.equals(
         Arrays.copyOfRange(data, start, start + length), readData2));
@@ -513,7 +513,8 @@ public class TestContainerPersistence {
     // Request to read the whole data in a single go.
     ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0,
         datalen * chunkCount);
-    byte[] newdata = chunkManager.readChunk(container, blockID, largeChunk);
+    byte[] newdata =
+        chunkManager.readChunk(container, blockID, largeChunk, false);
     MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
     newSha.update(newdata);
     Assert.assertEquals(Hex.encodeHexString(oldSha.digest()),
@@ -543,7 +544,7 @@ public class TestContainerPersistence {
     chunkManager.deleteChunk(container, blockID, info);
     exception.expect(StorageContainerException.class);
     exception.expectMessage("Unable to find the chunk file.");
-    chunkManager.readChunk(container, blockID, info);
+    chunkManager.readChunk(container, blockID, info, false);
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[35/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager Delegation Token support. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 904e597..c1695f1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -17,16 +17,23 @@
  */
 package org.apache.hadoop.ozone;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.slf4j.event.Level.INFO;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.PrivilegedExceptionAction;
 import java.util.Properties;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,14 +42,29 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.server.SCMStorage;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyPEMWriter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMStorage;
 import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.apache.hadoop.test.LambdaTestUtils;
@@ -52,6 +74,7 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
+import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,6 +84,8 @@ import org.slf4j.LoggerFactory;
 @InterfaceAudience.Private
 public final class TestSecureOzoneCluster {
 
+  private static final String TEST_USER = "testUgiUser";
+  private static final int CLIENT_TIMEOUT = 2 * 1000;
   private Logger LOGGER = LoggerFactory
       .getLogger(TestSecureOzoneCluster.class);
 
@@ -81,14 +106,24 @@ public final class TestSecureOzoneCluster {
   private static String clusterId;
   private static String scmId;
   private static String omId;
+  private OzoneManagerProtocolClientSideTranslatorPB omClient;
+  private KeyPair keyPair;
+  private Path metaDirPath;
 
   @Before
   public void init() {
     try {
       conf = new OzoneConfiguration();
+      conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+      DefaultMetricsSystem.setMiniClusterMode(true);
+      final String path = GenericTestUtils
+          .getTempPath(UUID.randomUUID().toString());
+      metaDirPath = Paths.get(path, "om-meta");
+      conf.set(OZONE_METADATA_DIRS, metaDirPath.toString());
       startMiniKdc();
       setSecureConfig(conf);
       createCredentialsInKDC(conf, miniKdc);
+      generateKeyPair(conf);
     } catch (IOException e) {
       LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
     } catch (Exception e) {
@@ -106,6 +141,10 @@ public final class TestSecureOzoneCluster {
       if (om != null) {
         om.stop();
       }
+      if (omClient != null) {
+        omClient.close();
+      }
+      FileUtils.deleteQuietly(metaDirPath.toFile());
     } catch (Exception e) {
       LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
     }
@@ -115,11 +154,11 @@ public final class TestSecureOzoneCluster {
       throws Exception {
     createPrincipal(scmKeytab,
         conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
-     createPrincipal(spnegoKeytab,
-         conf.get(ScmConfigKeys
-             .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
-        conf.get(OMConfigKeys
-            .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
+    createPrincipal(spnegoKeytab,
+        conf.get(ScmConfigKeys
+            .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
+    conf.get(OMConfigKeys
+        .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
     createPrincipal(omKeyTab,
         conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
   }
@@ -137,13 +176,14 @@ public final class TestSecureOzoneCluster {
     miniKdc.start();
   }
 
-  private void stopMiniKdc() throws Exception {
+  private void stopMiniKdc() {
     miniKdc.stop();
   }
 
   private void setSecureConfig(Configuration conf) throws IOException {
     conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
-    String host = KerberosUtil.getLocalHostName();
+    conf.setBoolean(OZONE_ENABLED, true);
+    String host = InetAddress.getLocalHost().getCanonicalHostName();
     String realm = miniKdc.getRealm();
     curUser = UserGroupInformation.getCurrentUser()
         .getUserName();
@@ -195,7 +235,7 @@ public final class TestSecureOzoneCluster {
     final String path = GenericTestUtils
         .getTempPath(UUID.randomUUID().toString());
     Path scmPath = Paths.get(path, "scm-meta");
-    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+    conf.set(OZONE_METADATA_DIRS, scmPath.toString());
     conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
     SCMStorage scmStore = new SCMStorage(conf);
     scmStore.setClusterId(clusterId);
@@ -245,60 +285,261 @@ public final class TestSecureOzoneCluster {
   }
 
   /**
-   * Tests the secure KSM Initialization Failure.
+   * Tests the secure om Initialization Failure.
    *
    * @throws IOException
    */
   @Test
-  public void testSecureKsmInitializationFailure() throws Exception {
+  public void testSecureOMInitializationFailure() throws Exception {
     initSCM();
     // Create a secure SCM instance as om client will connect to it
     scm = StorageContainerManager.createSCM(null, conf);
 
-    final String path = GenericTestUtils
-        .getTempPath(UUID.randomUUID().toString());
-    OMStorage ksmStore = new OMStorage(conf);
-    ksmStore.setClusterId("testClusterId");
-    ksmStore.setScmId("testScmId");
-    // writes the version file properties
-    ksmStore.initialize();
+    setupOm(conf);
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
         "non-existent-user@EXAMPLE.com");
     testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
   }
 
   /**
-   * Tests the secure KSM Initialization success.
+   * Tests the secure om Initialization success.
    *
    * @throws IOException
    */
   @Test
-  public void testSecureKsmInitializationSuccess() throws Exception {
+  public void testSecureOmInitializationSuccess() throws Exception {
     initSCM();
     // Create a secure SCM instance as om client will connect to it
     scm = StorageContainerManager.createSCM(null, conf);
     LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+    GenericTestUtils.setLogLevel(OzoneManager.LOG, INFO);
+
+    setupOm(conf);
+    try {
+      om.start();
+    } catch (Exception ex) {
+      // Expects timeout failure from scmClient in om but om user login via
+      // kerberos should succeed.
+      Assert.assertTrue(logs.getOutput().contains("Ozone Manager login"
+          + " successful"));
+    }
+  }
+
+  /**
+   * Performs following tests for delegation token.
+   * 1. Get valid delegation token
+   * 2. Test successful token renewal.
+   * 3. Client can authenticate using token.
+   * 4. Delegation token renewal without Kerberos auth fails.
+   * 5. Test success of token cancellation.
+   * 5. Test failure of token cancellation.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testDelegationToken() throws Exception {
+
+    // Capture logs for assertions
+    LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
     GenericTestUtils
-        .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
-            org.slf4j.event.Level.INFO);
+        .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+    // Setup secure OM for start
+    setupOm(conf);
+    long omVersion =
+        RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+    // Start OM
+    om.start();
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String username = ugi.getUserName();
+    ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+
+    // Get first OM client which will authenticate via Kerberos
+    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+        RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+            OmUtils.getOmAddress(conf), ugi, conf,
+            NetUtils.getDefaultSocketFactory(conf),
+            CLIENT_TIMEOUT));
+
+    // Assert if auth was successful via Kerberos
+    Assert.assertFalse(logs.getOutput().contains(
+        "Auth successful for " + username + " (auth:KERBEROS)"));
+
+    // Case 1: Test successful delegation token.
+    Token<OzoneTokenIdentifier> token = omClient
+        .getDelegationToken(new Text("om"));
+
+    // Case 2: Test successful token renewal.
+    long renewalTime = omClient.renewDelegationToken(token);
+    Assert.assertTrue(renewalTime > 0);
+
+    // Check if token is of right kind and renewer is running om instance
+    Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+    Assert.assertEquals(token.getService().toString(),
+        OmUtils.getOmRpcAddress(conf));
+    omClient.close();
+
+    // Create a remote ugi and set its authentication method to Token
+    UserGroupInformation testUser = UserGroupInformation
+        .createRemoteUser(TEST_USER);
+    testUser.addToken(token);
+    testUser.setAuthenticationMethod(AuthMethod.TOKEN);
+    UserGroupInformation.setLoginUser(testUser);
+
+    // Get Om client, this time authentication should happen via Token
+    testUser.doAs(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+            RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+                OmUtils.getOmAddress(conf), testUser, conf,
+                NetUtils.getDefaultSocketFactory(conf), CLIENT_TIMEOUT));
+        return null;
+      }
+    });
+
+    // Case 3: Test Client can authenticate using token.
+    Assert.assertFalse(logs.getOutput().contains(
+        "Auth successful for " + username + " (auth:TOKEN)"));
+    LambdaTestUtils.intercept(IOException.class, "Delete Volume failed," +
+            " error:VOLUME_NOT_FOUND",
+        () -> omClient.deleteVolume("vol1"));
+    Assert.assertTrue(logs.getOutput().contains(
+        "Auth successful for " + username + " (auth:TOKEN)"));
+
+    // Case 4: Test failure of token renewal.
+    // Call to renewDelegationToken will fail but it will confirm that
+    // initial connection via DT succeeded
+    LambdaTestUtils.intercept(RemoteException.class, "Delegation "
+            + "Token can be renewed only with kerberos or web authentication",
+        () -> omClient.renewDelegationToken(token));
+    Assert.assertTrue(logs.getOutput().contains(
+        "Auth successful for " + username + " (auth:TOKEN)"));
+    //testUser.setAuthenticationMethod(AuthMethod.KERBEROS);
+    UserGroupInformation.setLoginUser(ugi);
+    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+        RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+            OmUtils.getOmAddress(conf), ugi, conf,
+            NetUtils.getDefaultSocketFactory(conf),
+            Client.getRpcTimeout(conf)));
+
+    // Case 5: Test success of token cancellation.
+    omClient.cancelDelegationToken(token);
+    omClient.close();
+
+    // Wait for client to timeout
+    Thread.sleep(CLIENT_TIMEOUT);
+
+    Assert.assertFalse(logs.getOutput().contains("Auth failed for"));
+
+    // Case 6: Test failure of token cancellation.
+    // Get Om client, this time authentication using Token will fail as
+    // token is expired
+    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+        RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+            OmUtils.getOmAddress(conf), testUser, conf,
+            NetUtils.getDefaultSocketFactory(conf),
+            Client.getRpcTimeout(conf)));
+    LambdaTestUtils.intercept(RemoteException.class, "can't be found in cache",
+        () -> omClient.cancelDelegationToken(token));
+    Assert.assertTrue(logs.getOutput().contains(
+        "Auth failed for"));
+  }
 
-    final String path = GenericTestUtils
-        .getTempPath(UUID.randomUUID().toString());
-    Path metaDirPath = Paths.get(path, "om-meta");
+  private void generateKeyPair(OzoneConfiguration config) throws Exception {
+    HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf);
+    keyPair = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(config);
+    pemWriter.writeKey(keyPair, true);
+  }
 
-    OMStorage omStore = new OMStorage(conf);
+  /**
+   * Tests delegation token renewal.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testDelegationTokenRenewal() throws Exception {
+    // Capture logs for assertions.
+    LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
+    GenericTestUtils
+        .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+    // Setup secure OM for start.
+    OzoneConfiguration newConf = new OzoneConfiguration(conf);
+    newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 500);
+    setupOm(newConf);
+    long omVersion =
+        RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+    OzoneManager.setTestSecureOmFlag(true);
+    // Start OM
+
+    om.start();
+
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String username = ugi.getUserName();
+
+    // Get first OM client which will authenticate via Kerberos
+    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+        RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+            OmUtils.getOmAddress(conf), ugi, conf,
+            NetUtils.getDefaultSocketFactory(conf),
+            CLIENT_TIMEOUT));
+
+    // Since client is already connected get a delegation token
+    Token<OzoneTokenIdentifier> token = omClient
+        .getDelegationToken(new Text("om"));
+
+    // Check if token is of right kind and renewer is running om instance
+    Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+    Assert.assertEquals(token.getService().toString(),
+        OmUtils.getOmRpcAddress(conf));
+
+    // Renew delegation token
+    long expiryTime = omClient.renewDelegationToken(token);
+    Assert.assertTrue(expiryTime > 0);
+
+    // Test failure of delegation renewal
+    // 1. When renewer doesn't match (implicitly covers when renewer is
+    // null or empty )
+    Token token2 = omClient.getDelegationToken(new Text("randomService"));
+    LambdaTestUtils.intercept(RemoteException.class,
+        " with non-matching renewer randomService",
+        () -> omClient.renewDelegationToken(token2));
+
+    // 2. Test tampered token
+    OzoneTokenIdentifier tokenId = OzoneTokenIdentifier
+        .readProtoBuf(token.getIdentifier());
+    tokenId.setRenewer(new Text("om"));
+    tokenId.setMaxDate(System.currentTimeMillis() * 2);
+    Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
+        tokenId.getBytes(), token2.getPassword(), token2.getKind(),
+        token2.getService());
+    LambdaTestUtils
+        .intercept(RemoteException.class, "can't be found in cache",
+            () -> omClient.renewDelegationToken(tamperedToken));
+
+    // 3. When token maxExpiryTime exceeds
+    Thread.sleep(500);
+    LambdaTestUtils
+        .intercept(RemoteException.class, "om tried to renew an expired"
+            + " token", () -> omClient.renewDelegationToken(token));
+  }
+
+  private void setupOm(OzoneConfiguration config) throws Exception {
+    OMStorage omStore = new OMStorage(config);
     omStore.setClusterId("testClusterId");
     omStore.setScmId("testScmId");
     // writes the version file properties
     omStore.initialize();
-    try {
-      om = OzoneManager.createOm(null, conf);
-    } catch (Exception ex) {
-      // Expects timeout failure from scmClient in KSM but KSM user login via
-      // kerberos should succeed
-      Assert.assertTrue(
-          logs.getOutput().contains("Ozone Manager login successful."));
-    }
+    OzoneManager.setTestSecureOmFlag(true);
+    om = OzoneManager.createOm(null, config);
+    CertificateClient certClient = Mockito.mock(CertificateClient.class);
+    Mockito.when(certClient.getPrivateKey("om"))
+        .thenReturn(keyPair.getPrivate());
+    Mockito.when(certClient.getPublicKey("om"))
+        .thenReturn(keyPair.getPublic());
+    om.setCertClient(certClient);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a5cec20..c6616aa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.protobuf.BlockingService;
+import java.security.KeyPair;
+import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -35,12 +37,20 @@ import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideT
 import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
 import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
 import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ipc.Client;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.security.OzoneSecurityException;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.security.OzoneSecretManager;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.NetUtils;
@@ -53,7 +63,6 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
 import org.apache.hadoop.ozone.audit.AuditMessage;
 import org.apache.hadoop.ozone.audit.Auditor;
 import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -74,6 +83,8 @@ import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
@@ -99,6 +110,7 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.hadoop.ozone.security.OzoneSecurityException.ResultCodes.*;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
 import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
 import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
@@ -134,13 +146,21 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   public static final Logger LOG =
       LoggerFactory.getLogger(OzoneManager.class);
 
-  private static final AuditLogger AUDIT =
-      new AuditLogger(AuditLoggerType.OMLOGGER);
+  private static final AuditLogger AUDIT = new AuditLogger(
+      AuditLoggerType.OMLOGGER);
 
   private static final String USAGE =
       "Usage: \n ozone om [genericOptions] " + "[ "
           + StartupOption.INIT.getName() + " ]\n " + "ozone om [ "
           + StartupOption.HELP.getName() + " ]\n";
+  private static final String OM_DAEMON = "om";
+  private static boolean securityEnabled = false;
+  private static OzoneSecretManager secretManager;
+  // TO DO: For testing purpose only, remove before commiting
+  private KeyPair keyPair;
+  private CertificateClient certClient;
+  private static boolean testSecureOmFlag = false;
+  private final Text omRpcAddressTxt;
   private final OzoneConfiguration configuration;
   private RPC.Server omRpcServer;
   private InetSocketAddress omRpcAddress;
@@ -174,21 +194,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
           ResultCodes.OM_NOT_INITIALIZED);
     }
 
-    scmContainerClient = getScmContainerClient(configuration);
-
-    // verifies that the SCM info in the OM Version file is correct.
-    scmBlockClient = getScmBlockClient(configuration);
-
-    ScmInfo scmInfo = scmBlockClient.getScmInfo();
-    if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
-        .getScmId().equals(omStorage.getScmId()))) {
-      throw new OMException("SCM version info mismatch.",
-          ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+    if (!testSecureOmFlag) {
+      scmContainerClient = getScmContainerClient(configuration);
+      // verifies that the SCM info in the OM Version file is correct.
+      scmBlockClient = getScmBlockClient(configuration);
+      ScmInfo scmInfo = scmBlockClient.getScmInfo();
+      if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
+          .getScmId().equals(omStorage.getScmId()))) {
+        throw new OMException("SCM version info mismatch.",
+            ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+      }
+    } else {
+      // For testing purpose only
+      scmContainerClient = null;
+      scmBlockClient = null;
     }
+    final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
+        OZONE_OM_HANDLER_COUNT_DEFAULT);
 
     RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
         ProtobufRpcEngine.class);
 
+    BlockingService omService = newReflectiveBlockingService(
+        new OzoneManagerProtocolServerSideTranslatorPB(this));
+    final InetSocketAddress omNodeRpcAddr =
+        getOmAddress(configuration);
+    omRpcAddressTxt = new Text(OmUtils.getOmRpcAddress(configuration));
+    secretManager = createSecretManager(configuration);
+
+    omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
+        OzoneManagerProtocolPB.class, omService,
+        handlerCount);
+    omRpcAddress = updateRPCListenAddress(configuration,
+        OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
     metadataManager = new OmMetadataManagerImpl(configuration);
     volumeManager = new VolumeManagerImpl(metadataManager, configuration);
     bucketManager = new BucketManagerImpl(metadataManager);
@@ -258,6 +296,65 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   }
 
 
+  private OzoneSecretManager createSecretManager(
+      OzoneConfiguration conf)
+      throws IOException {
+    long tokenRemoverScanInterval =
+        conf.getTimeDuration(OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY,
+            OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT,
+            TimeUnit.MILLISECONDS);
+    long tokenMaxLifetime =
+        conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+            OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+            TimeUnit.MILLISECONDS);
+    long tokenRenewInterval =
+        conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+            OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
+            TimeUnit.MILLISECONDS);
+    return new OzoneSecretManager(conf, tokenMaxLifetime, tokenRenewInterval,
+        tokenRemoverScanInterval, omRpcAddressTxt);
+  }
+
+  private void stopSecretManager() throws IOException {
+    if (secretManager != null) {
+      LOG.info("Stopping OM secret manager");
+      secretManager.stop();
+    }
+  }
+
+  private void startSecretManager() {
+    if (secretManager != null) {
+      try {
+        readKeyPair();
+        LOG.info("Starting OM secret manager");
+        secretManager.startThreads(keyPair);
+      } catch (IOException e) {
+        // Inability to start secret manager
+        // can't be recovered from.
+        LOG.error("Error starting secret manager.", e);
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  public void setCertClient(CertificateClient certClient) {
+    // TODO: Initialize it in contructor with implementation for certClient.
+    this.certClient = certClient;
+  }
+
+  /**
+   * Read private key from file.
+   */
+  private void readKeyPair() throws OzoneSecurityException {
+    try {
+      keyPair = new KeyPair(certClient.getPublicKey(OM_DAEMON),
+          certClient.getPrivateKey(OM_DAEMON));
+    } catch (Exception e) {
+      throw new OzoneSecurityException("Error reading private file for "
+          + "OzoneManager", e, OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
+    }
+  }
+
   /**
    * Login OM service user if security and Kerberos are enabled.
    *
@@ -267,8 +364,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private static void loginOMUser(OzoneConfiguration conf)
       throws IOException, AuthenticationException {
 
-    if (SecurityUtil.getAuthenticationMethod(conf).equals
-        (AuthenticationMethod.KERBEROS)) {
+    if (SecurityUtil.getAuthenticationMethod(conf).equals(
+        AuthenticationMethod.KERBEROS)) {
       LOG.debug("Ozone security is enabled. Attempting login for OM user. "
               + "Principal: {},keytab: {}", conf.get(
           OZONE_OM_KERBEROS_PRINCIPAL_KEY),
@@ -280,8 +377,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
           OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
     } else {
-      throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
-          (conf) + " authentication method not supported. OM user login "
+      throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+          conf) + " authentication method not supported. OM user login "
           + "failed.");
     }
     LOG.info("Ozone Manager login successful.");
@@ -354,7 +451,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
         .setPort(addr.getPort())
         .setNumHandlers(handlerCount)
         .setVerbose(false)
-        .setSecretManager(null)
+        .setSecretManager(secretManager)
         .build();
 
     DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
@@ -394,6 +491,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     out.println(USAGE + "\n");
   }
 
+  private static boolean isOzoneSecurityEnabled() {
+    return securityEnabled;
+  }
+
   /**
    * Constructs OM instance based on command line arguments.
    *
@@ -438,8 +539,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       terminate(1);
       return null;
     }
+
+    securityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
     // Authenticate KSM if security is enabled
-    if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+    if (securityEnabled) {
       loginOMUser(conf);
     }
     switch (startOpt) {
@@ -572,19 +675,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
    * Start service.
    */
   public void start() throws IOException {
-
-    InetSocketAddress omNodeRpcAddr = getOmAddress(configuration);
-    int handlerCount = configuration.getInt(OZONE_OM_HANDLER_COUNT_KEY,
-        OZONE_OM_HANDLER_COUNT_DEFAULT);
-    BlockingService omService = newReflectiveBlockingService(
-        new OzoneManagerProtocolServerSideTranslatorPB(this));
-    omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
-        OzoneManagerProtocolPB.class, omService,
-        handlerCount);
-    omRpcAddress = updateRPCListenAddress(configuration,
-        OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
-    omRpcServer.start();
-
     LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
         omRpcAddress));
 
@@ -592,7 +682,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     DefaultMetricsSystem.initialize("OzoneManager");
 
     metadataManager.start(configuration);
-
+    startSecretManagerIfNecessary();
 
     // Set metrics and start metrics back ground thread
     metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager
@@ -613,8 +703,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     metricsTimer.schedule(scheduleOMMetricsWriteTask, 0, period);
 
     keyManager.start(configuration);
-
-    httpServer = new OzoneManagerHttpServer(configuration, this);
+    omRpcServer.start();
     try {
       httpServer.start();
     } catch (Exception ex) {
@@ -636,6 +725,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       scheduleOMMetricsWriteTask = null;
       omRpcServer.stop();
       keyManager.stop();
+      stopSecretManager();
       httpServer.stop();
       metadataManager.stop();
       metrics.unRegister();
@@ -657,6 +747,140 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     }
   }
 
+  private void startSecretManagerIfNecessary() {
+    boolean shouldRun = shouldUseDelegationTokens() && isOzoneSecurityEnabled();
+    boolean running = secretManager.isRunning();
+    if (shouldRun && !running) {
+      startSecretManager();
+    }
+  }
+
+  private boolean shouldUseDelegationTokens() {
+    return UserGroupInformation.isSecurityEnabled();
+  }
+
+
+  /**
+   *
+   * @return true if delegation token operation is allowed
+   */
+  private boolean isAllowedDelegationTokenOp() throws IOException {
+    AuthenticationMethod authMethod = getConnectionAuthenticationMethod();
+    if (UserGroupInformation.isSecurityEnabled()
+        && (authMethod != AuthenticationMethod.KERBEROS)
+        && (authMethod != AuthenticationMethod.KERBEROS_SSL)
+        && (authMethod != AuthenticationMethod.CERTIFICATE)) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Returns authentication method used to establish the connection.
+   * @return AuthenticationMethod used to establish connection
+   * @throws IOException
+   */
+  private AuthenticationMethod getConnectionAuthenticationMethod()
+      throws IOException {
+    UserGroupInformation ugi = getRemoteUser();
+    AuthenticationMethod authMethod = ugi.getAuthenticationMethod();
+    if (authMethod == AuthenticationMethod.PROXY) {
+      authMethod = ugi.getRealUser().getAuthenticationMethod();
+    }
+    return authMethod;
+  }
+
+  // optimize ugi lookup for RPC operations to avoid a trip through
+  // UGI.getCurrentUser which is synch'ed
+  private static UserGroupInformation getRemoteUser() throws IOException {
+    UserGroupInformation ugi = Server.getRemoteUser();
+    return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
+  }
+
+  /**
+   * Get delegation token from OzoneManager.
+   * @param renewer Renewer information
+   * @return delegationToken DelegationToken signed by OzoneManager
+   * @throws IOException on error
+   */
+  @Override
+  public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException {
+    final boolean success;
+    final String tokenId;
+    Token<OzoneTokenIdentifier> token;
+
+    if (!isAllowedDelegationTokenOp()) {
+      throw new IOException("Delegation Token can be issued only with "
+          + "kerberos or web authentication");
+    }
+    if (secretManager == null || !secretManager.isRunning()) {
+      LOG.warn("trying to get DT with no secret manager running in OM.");
+      return null;
+    }
+
+    UserGroupInformation ugi = getRemoteUser();
+    String user = ugi.getUserName();
+    Text owner = new Text(user);
+    Text realUser = null;
+    if (ugi.getRealUser() != null) {
+      realUser = new Text(ugi.getRealUser().getUserName());
+    }
+
+    token = secretManager.createToken(owner, renewer, realUser);
+    return token;
+  }
+
+  /**
+   * Method to renew a delegationToken issued by OzoneManager.
+   * @param token token to renew
+   * @return new expiryTime of the token
+   * @throws InvalidToken if {@code token} is invalid
+   * @throws IOException on other errors
+   */
+  @Override
+  public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws InvalidToken, IOException {
+    long expiryTime;
+
+    try {
+
+      if (!isAllowedDelegationTokenOp()) {
+        throw new IOException("Delegation Token can be renewed only with "
+            + "kerberos or web authentication");
+      }
+      String renewer = getRemoteUser().getShortUserName();
+      expiryTime = secretManager.renewToken(token, renewer);
+
+    } catch (AccessControlException ace) {
+      final OzoneTokenIdentifier id = OzoneTokenIdentifier.readProtoBuf(
+          token.getIdentifier());
+      LOG.error("Delegation token renewal failed for dt: {}, cause: {}",
+          id.toString(), ace.getMessage());
+      throw ace;
+    }
+    return expiryTime;
+  }
+
+  /**
+   * Cancels a delegation token.
+   * @param token token to cancel
+   * @throws IOException on error
+   */
+  @Override
+  public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    OzoneTokenIdentifier id = null;
+    try {
+      String canceller = getRemoteUser().getUserName();
+      id = secretManager.cancelToken(token, canceller);
+      LOG.trace("Delegation token renewed for dt: {}", id);
+    } catch (AccessControlException ace) {
+      LOG.error("Delegation token renewal failed for dt: {}, cause: {}", id,
+          ace.getMessage());
+      throw ace;
+    }
+  }
   /**
    * Creates a volume.
    *
@@ -1430,4 +1654,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       return name;
     }
   }
+
+  public static void setTestSecureOmFlag(boolean testSecureOmFlag) {
+    OzoneManager.testSecureOmFlag = testSecureOmFlag;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 64806ed..e2536d3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -119,6 +119,15 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .SetVolumePropertyResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
     .Status;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -651,4 +660,54 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
     }
     return resp.build();
   }
+
+  @Override
+  public GetDelegationTokenResponseProto getDelegationToken(
+      RpcController controller, GetDelegationTokenRequestProto request)
+      throws ServiceException {
+    try {
+      Token<OzoneTokenIdentifier> token = impl
+          .getDelegationToken(new Text(request.getRenewer()));
+      if (token != null) {
+        return GetDelegationTokenResponseProto.newBuilder()
+            .setToken(OMPBHelper.convertToTokenProto(token)).build();
+      }
+      return GetDelegationTokenResponseProto.getDefaultInstance();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public RenewDelegationTokenResponseProto renewDelegationToken(
+      RpcController controller, RenewDelegationTokenRequestProto request)
+      throws ServiceException {
+    try {
+      if(request.hasToken()) {
+        long expiryTime = impl
+            .renewDelegationToken(
+                OMPBHelper.convertToDelegationToken(request.getToken()));
+        return RenewDelegationTokenResponseProto.newBuilder()
+            .setNewExpiryTime(expiryTime).build();
+      }
+      return RenewDelegationTokenResponseProto.getDefaultInstance();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public CancelDelegationTokenResponseProto cancelDelegationToken(
+      RpcController controller, CancelDelegationTokenRequestProto req)
+      throws ServiceException {
+    try {
+      if(req.hasToken()) {
+        impl.cancelDelegationToken(
+            OMPBHelper.convertToDelegationToken(req.getToken()));
+      }
+      return CancelDelegationTokenResponseProto.getDefaultInstance();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[46/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-751.

Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-751.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bee23d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bee23d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bee23d2

Branch: refs/heads/HDDS-4
Commit: 3bee23d24f6d90592e715be35fa6213a2723e0a6
Parents: 18c9db9
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:38:30 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bee23d2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index e355d0a..1bb9972 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -176,9 +176,11 @@ public final class HddsServerUtil {
         ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
 
     return NetUtils.createSocketAddr(
-        host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+        host.orElse(ScmConfigKeys
+            .OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
             ":" + port
-            .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+            .orElse(conf.getInt(ScmConfigKeys
+                    .OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
                 ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[27/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed Certificate. Contributed by Anu Engineer.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..db5d430
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private OzoneConfiguration configuration;
+  private HDDSKeyGenerator keyGenerator;
+  private String prefix;
+
+  @Before
+  public void init() throws IOException {
+    configuration = new OzoneConfiguration();
+    prefix = temporaryFolder.newFolder().toString();
+    configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+    keyGenerator = new HDDSKeyGenerator(configuration);
+  }
+
+  /**
+   * Assert basic things like we are able to create a file, and the names are
+   * in expected format etc.
+   *
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   * @throws IOException - On I/O failure.
+   */
+  @Test
+  public void testWriteKey()
+      throws NoSuchProviderException, NoSuchAlgorithmException,
+      IOException, InvalidKeySpecException {
+    KeyPair keys = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    pemWriter.writeKey(keys);
+
+    // Assert that locations have been created.
+    Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+    Assert.assertTrue(keyLocation.toFile().exists());
+
+    // Assert that locations are created in the locations that we specified
+    // using the Config.
+    Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+    Path privateKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPrivateKeyFileName());
+    Assert.assertTrue(privateKeyPath.toFile().exists());
+    Path publicKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPublicKeyFileName());
+    Assert.assertTrue(publicKeyPath.toFile().exists());
+
+    // Read the private key and test if the expected String in the PEM file
+    // format exists.
+    byte[] privateKey = Files.readAllBytes(privateKeyPath);
+    String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+    // Read the public key and test if the expected String in the PEM file
+    // format exists.
+    byte[] publicKey = Files.readAllBytes(publicKeyPath);
+    String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+    // Let us decode the PEM file and parse it back into binary.
+    KeyFactory kf = KeyFactory.getInstance(
+        pemWriter.getSecurityConfig().getKeyAlgo());
+
+    // Replace the PEM Human readable guards.
+    privateKeydata =
+        privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+    privateKeydata =
+        privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+    // Decode the bas64 to binary format and then use an ASN.1 parser to
+    // parse the binary format.
+
+    byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+    PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+    PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+    Assert.assertNotNull("Private Key should not be null",
+        privateKeyDecoded);
+
+    // Let us decode the public key and veriy that we can parse it back into
+    // binary.
+    publicKeydata =
+        publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+    publicKeydata =
+        publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+    keyBytes = Base64.decodeBase64(publicKeydata);
+    X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+    PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+    Assert.assertNotNull("Public Key should not be null",
+        publicKeyDecoded);
+
+    // Now let us assert the permissions on the Directories and files are as
+    // expected.
+    Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+    Set<PosixFilePermission> currentSet =
+        Files.getPosixFilePermissions(privateKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(publicKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(keyLocation);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+  }
+
+  /**
+   * Assert key rewrite fails without force option.
+   *
+   * @throws IOException - on I/O failure.
+   */
+  @Test
+  public void testReWriteKey()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    SecurityConfig secConfig = pemWriter.getSecurityConfig();
+    pemWriter.writeKey(kp);
+
+    // Assert that rewriting of keys throws exception with valid messages.
+    LambdaTestUtils
+        .intercept(IOException.class, "Private Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPrivateKeyFileName()).toFile());
+    LambdaTestUtils
+        .intercept(IOException.class, "Public Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPublicKeyFileName()).toFile());
+
+    // Should succeed now as both public and private key are deleted.
+    pemWriter.writeKey(kp);
+    // Should succeed with overwrite flag as true.
+    pemWriter.writeKey(kp, true);
+
+  }
+
+  /**
+   * Assert key rewrite fails in non Posix file system.
+   *
+   * @throws IOException - on I/O failure.
+   */
+  @Test
+  public void testWriteKeyInNonPosixFS()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    pemWriter.setIsPosixFileSystem(() -> false);
+
+    // Assert key rewrite fails in non Posix file system.
+    LambdaTestUtils
+        .intercept(IOException.class, "Unsupported File System for pem file.",
+            () -> pemWriter.writeKey(kp));
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..49e40b4
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test package for keys used in X.509 env.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..f541468
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * X.509 Certificate and keys related tests.
+ */
+package org.apache.hadoop.hdds.security.x509;
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[14/50] [abbrv] hadoop git commit: HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.

Posted by xy...@apache.org.
HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5347368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5347368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5347368

Branch: refs/heads/HDDS-4
Commit: f534736867eed962899615ca1b7eb68bcf591d17
Parents: d0edd37
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Nov 30 00:18:27 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Nov 30 00:18:27 2018 +0530

----------------------------------------------------------------------
 .../hadoop/hdfs/DFSOpsCountStatistics.java      |  9 +++
 .../hadoop/hdfs/DistributedFileSystem.java      | 18 ++++++
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 63 +++++++++++++++++++-
 3 files changed, 89 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index 3dcf13b..b9852ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
 
   /** This is for counting distributed file system operations. */
   public enum OpType {
+    ADD_EC_POLICY("op_add_ec_policy"),
     ALLOW_SNAPSHOT("op_allow_snapshot"),
     APPEND(CommonStatisticNames.OP_APPEND),
     CONCAT("op_concat"),
@@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics {
     CREATE_SYM_LINK("op_create_symlink"),
     DELETE(CommonStatisticNames.OP_DELETE),
     DELETE_SNAPSHOT("op_delete_snapshot"),
+    DISABLE_EC_POLICY("op_disable_ec_policy"),
     DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+    ENABLE_EC_POLICY("op_enable_ec_policy"),
     EXISTS(CommonStatisticNames.OP_EXISTS),
     GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
     GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+    GET_EC_CODECS("op_get_ec_codecs"),
+    GET_EC_POLICY("op_get_ec_policy"),
+    GET_EC_POLICIES("op_get_ec_policies"),
     GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
     GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
     GET_FILE_LINK_STATUS("op_get_file_link_status"),
@@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics {
     REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
     REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
     REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+    REMOVE_EC_POLICY("op_remove_ec_policy"),
     REMOVE_XATTR("op_remove_xattr"),
     RENAME(CommonStatisticNames.OP_RENAME),
     RENAME_SNAPSHOT("op_rename_snapshot"),
     RESOLVE_LINK("op_resolve_link"),
     SET_ACL(CommonStatisticNames.OP_SET_ACL),
+    SET_EC_POLICY("op_set_ec_policy"),
     SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
     SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
     SET_REPLICATION("op_set_replication"),
@@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
     GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
     GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"),
     TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+    UNSET_EC_POLICY("op_unset_ec_policy"),
     UNSET_STORAGE_POLICY("op_unset_storage_policy");
 
     private static final Map<String, OpType> SYMBOL_MAP =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index ca1546c..7dd02bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2845,6 +2845,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public void setErasureCodingPolicy(final Path path,
       final String ecPolicyName) throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY);
     Path absF = fixRelativePart(path);
     new FileSystemLinkResolver<Void>() {
       @Override
@@ -2912,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
       throws IOException {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY);
     Path absF = fixRelativePart(path);
     return new FileSystemLinkResolver<ErasureCodingPolicy>() {
       @Override
@@ -2943,6 +2947,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
       throws IOException {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES);
     return Arrays.asList(dfs.getErasureCodingPolicies());
   }
 
@@ -2955,6 +2961,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public Map<String, String> getAllErasureCodingCodecs()
       throws IOException {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS);
     return dfs.getErasureCodingCodecs();
   }
 
@@ -2971,6 +2979,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies)  throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY);
     return dfs.addErasureCodingPolicies(policies);
   }
 
@@ -2982,6 +2992,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public void removeErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY);
     dfs.removeErasureCodingPolicy(ecPolicyName);
   }
 
@@ -2993,6 +3005,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public void enableErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY);
     dfs.enableErasureCodingPolicy(ecPolicyName);
   }
 
@@ -3004,6 +3018,8 @@ public class DistributedFileSystem extends FileSystem
    */
   public void disableErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY);
     dfs.disableErasureCodingPolicy(ecPolicyName);
   }
 
@@ -3014,6 +3030,8 @@ public class DistributedFileSystem extends FileSystem
    * @throws IOException
    */
   public void unsetErasureCodingPolicy(final Path path) throws IOException {
+    statistics.incrementWriteOps(1);
+    storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY);
     Path absF = fixRelativePart(path);
     new FileSystemLinkResolver<Void>() {
       @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index cae0fbf..97ced7a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -773,7 +773,68 @@ public class TestDistributedFileSystem {
     } finally {
       if (cluster != null) cluster.shutdown();
     }
-    
+  }
+
+  @Test
+  public void testECStatistics() throws IOException {
+    try (MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(getTestConfiguration()).build()) {
+      cluster.waitActive();
+      final DistributedFileSystem dfs = cluster.getFileSystem();
+      Path dir = new Path("/test");
+      dfs.mkdirs(dir);
+      int readOps = 0;
+      int writeOps = 0;
+      FileSystem.clearStatistics();
+
+      long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY);
+      dfs.enableErasureCodingPolicy("RS-10-4-1024k");
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.SET_EC_POLICY);
+      dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k");
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.GET_EC_POLICY);
+      dfs.getErasureCodingPolicy(dir);
+      checkStatistics(dfs, ++readOps, writeOps, 0);
+      checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.UNSET_EC_POLICY);
+      dfs.unsetErasureCodingPolicy(dir);
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.GET_EC_POLICIES);
+      dfs.getAllErasureCodingPolicies();
+      checkStatistics(dfs, ++readOps, writeOps, 0);
+      checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1);
+
+      opCount = getOpStatistics(OpType.GET_EC_CODECS);
+      dfs.getAllErasureCodingCodecs();
+      checkStatistics(dfs, ++readOps, writeOps, 0);
+      checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1);
+
+      ErasureCodingPolicy newPolicy =
+          new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024);
+
+      opCount = getOpStatistics(OpType.ADD_EC_POLICY);
+      dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy});
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.REMOVE_EC_POLICY);
+      dfs.removeErasureCodingPolicy("RS-5-3-1024k");
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1);
+
+      opCount = getOpStatistics(OpType.DISABLE_EC_POLICY);
+      dfs.disableErasureCodingPolicy("RS-10-4-1024k");
+      checkStatistics(dfs, readOps, ++writeOps, 0);
+      checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1);
+    }
   }
 
   @SuppressWarnings("ThrowableResultOfMethodCallIgnored")


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[38/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-9. Add GRPC protocol interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96bd574d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96bd574d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96bd574d

Branch: refs/heads/HDDS-4
Commit: 96bd574dc9104ce017c889dea93792ac00900f01
Parents: 59767be
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 20 20:21:08 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../hdds/scm/ClientCredentialInterceptor.java   |  65 ++++
 .../hadoop/hdds/scm/XceiverClientGrpc.java      |  64 +++-
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |   5 +-
 .../exception/SCMSecurityException.java         |  52 +++
 .../hdds/security/exception/package-info.java   |  23 ++
 .../security/token/BlockTokenException.java     |  53 ++++
 .../hdds/security/token/BlockTokenVerifier.java | 113 +++++++
 .../token/OzoneBlockTokenIdentifier.java        | 199 ++++++++++++
 .../security/token/OzoneBlockTokenSelector.java |  55 ++++
 .../hdds/security/token/TokenVerifier.java      |  38 +++
 .../hdds/security/token/package-info.java       |  22 ++
 .../hdds/security/x509/SecurityConfig.java      |  11 +
 .../authority/CertificateServer.java            |   2 +-
 .../certificate/client/CertificateClient.java   |  11 +
 .../certificates/CertificateSignRequest.java    |   2 +-
 .../certificates/SelfSignedCertificate.java     |   2 +-
 .../x509/exceptions/CertificateException.java   |  14 +-
 .../x509/exceptions/SCMSecurityException.java   |  64 ----
 .../org/apache/hadoop/ozone/OzoneConsts.java    |  14 +
 hadoop-hdds/common/src/main/proto/hdds.proto    |   2 +-
 .../token/TestOzoneBlockTokenIdentifier.java    | 313 +++++++++++++++++++
 .../hdds/security/token/package-info.java       |  22 ++
 .../TestCertificateSignRequest.java             |   2 +-
 .../x509/certificates/TestRootCertificate.java  |   2 +-
 .../server/ServerCredentialInterceptor.java     |  74 +++++
 .../transport/server/XceiverServerGrpc.java     |  23 +-
 .../security/OzoneBlockTokenIdentifier.java     | 178 -----------
 .../ozone/security/OzoneBlockTokenSelector.java |  55 ----
 .../ozoneimpl/TestSecureOzoneContainer.java     | 209 +++++++++++++
 .../security/TestOzoneBlockTokenIdentifier.java | 255 ---------------
 30 files changed, 1363 insertions(+), 581 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
new file mode 100644
index 0000000..7a15808
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.ratis.thirdparty.io.grpc.CallOptions;
+import org.apache.ratis.thirdparty.io.grpc.Channel;
+import org.apache.ratis.thirdparty.io.grpc.ClientCall;
+import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+
+/**
+ * GRPC client interceptor for ozone block token.
+ */
+public class ClientCredentialInterceptor implements ClientInterceptor {
+
+  private final String user;
+  private final String token;
+
+  public ClientCredentialInterceptor(String user, String token) {
+    this.user = user;
+    this.token = token;
+  }
+
+  @Override
+  public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
+      MethodDescriptor<ReqT, RespT> method,
+      CallOptions callOptions,
+      Channel next) {
+
+    return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
+        next.newCall(method, callOptions)) {
+      @Override
+      public void start(Listener<RespT> responseListener, Metadata headers) {
+        if (token != null) {
+          headers.put(OBT_METADATA_KEY, token);
+        }
+        if (user != null) {
+          headers.put(USER_METADATA_KEY, user);
+        }
+        super.start(responseListener, headers);
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index a824c29..ccc9372 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -31,22 +31,32 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServi
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSelector;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Time;
 import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.Status;
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
 import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
-import java.util.UUID;
-import java.util.Map;
-import java.util.HashMap;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -93,7 +103,9 @@ public class XceiverClientGrpc extends XceiverClientSpi {
     connectToDatanode(dn);
   }
 
-  private void connectToDatanode(DatanodeDetails dn) {
+
+  private void connectToDatanode(DatanodeDetails dn) throws IOException,
+      SCMSecurityException {
     // read port from the data node, on failure use default configured
     // port.
     int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
@@ -101,16 +113,49 @@ public class XceiverClientGrpc extends XceiverClientSpi {
       port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
           OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
     }
+
+    // Add credential context to the client call
+    String userName = UserGroupInformation.getCurrentUser()
+        .getShortUserName();
+
+    // Add block token if block token (mutual auth) is required but the client
+    // does not have a mTLS (private key and ca signed certificate)
+    String encodedToken = null;
+    SecurityConfig secConfig = new SecurityConfig(config);
+    if (secConfig.isGrpcBlockTokenEnabled()) {
+      InetSocketAddress addr = new InetSocketAddress(dn.getIpAddress(), port);
+      encodedToken = getEncodedBlockToken(addr);
+      if (encodedToken == null) {
+        throw new SCMSecurityException("No Block token available to access " +
+            "service at : " + addr.toString());
+      }
+    }
     LOG.debug("Connecting to server Port : " + dn.getIpAddress());
-    ManagedChannel channel =
-        NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
+    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn
+            .getIpAddress(), port).usePlaintext()
             .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
-            .build();
+            .intercept(new ClientCredentialInterceptor(userName, encodedToken));
+    ManagedChannel channel = channelBuilder.build();
     XceiverClientProtocolServiceStub asyncStub =
         XceiverClientProtocolServiceGrpc.newStub(channel);
     asyncStubs.put(dn.getUuid(), asyncStub);
     channels.put(dn.getUuid(), channel);
   }
+
+  private String getEncodedBlockToken(InetSocketAddress addr)
+      throws IOException{
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    OzoneBlockTokenSelector tokenSelector = new OzoneBlockTokenSelector();
+    Text service = SecurityUtil.buildTokenService(addr);
+    Token<OzoneBlockTokenIdentifier> token = tokenSelector.selectToken(
+        service, ugi.getTokens());
+    if (token != null) {
+      token.setService(service);
+      return token.encodeToUrlString();
+    }
+    return null;
+  }
+
   /**
    * Returns if the xceiver client connects to all servers in the pipeline.
    *
@@ -173,6 +218,11 @@ public class XceiverClientGrpc extends XceiverClientSpi {
       } catch (ExecutionException | InterruptedException e) {
         LOG.warn("Failed to execute command " + request + " on datanode " + dn
             .getUuidString(), e);
+        if (Status.fromThrowable(e.getCause()).getCode()
+            == Status.UNAUTHENTICATED.getCode()) {
+          throw new SCMSecurityException("Failed to authenticate with " +
+              "GRPC XceiverServer with Ozone block token.");
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index ef1194b..553f4aa 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -145,6 +145,7 @@ public final class HddsConfigKeys {
       "hdds.x509.signature.algorithm";
   public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
 
-
-
+  public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED = "hdds.grpc.block" +
+      ".token.enabled";
+  public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
new file mode 100644
index 0000000..21bc45b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.exception;
+
+import java.io.IOException;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends IOException {
+
+  /**
+   * Ctor.
+   * @param message - Error Message.
+   */
+  public SCMSecurityException(String message) {
+    super(message);
+  }
+
+  /**
+   * Ctor.
+   * @param message - Message.
+   * @param cause  - Actual cause.
+   */
+  public SCMSecurityException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  /**
+   * Ctor.
+   * @param cause - Base Exception.
+   */
+  public SCMSecurityException(Throwable cause) {
+    super(cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
new file mode 100644
index 0000000..b980592
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by SCM security classes.
+ */
+package org.apache.hadoop.hdds.security.exception;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
new file mode 100644
index 0000000..7ea0ebc
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
+/**
+ * Block Token Exceptions from the SCM Security layer.
+ */
+public class BlockTokenException extends SCMSecurityException {
+
+  /**
+   * Ctor.
+   * @param message - Error Message.
+   */
+  public BlockTokenException(String message) {
+    super(message);
+  }
+
+  /**
+   * Ctor.
+   * @param message - Message.
+   * @param cause  - Actual cause.
+   */
+  public BlockTokenException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  /**
+   * Ctor.
+   * @param cause - Base Exception.
+   */
+  public BlockTokenException(Throwable cause) {
+    super(cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
new file mode 100644
index 0000000..8c0d19e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+
+
+/**
+ * Verify token and return a UGI with token if authenticated.
+ */
+public class BlockTokenVerifier implements TokenVerifier {
+
+  private final CertificateClient caClient;
+  private final SecurityConfig conf;
+
+  public BlockTokenVerifier(SecurityConfig conf, CertificateClient caClient) {
+    this.conf = conf;
+    this.caClient = caClient;
+  }
+
+  private boolean isExpired(long expiryDate) {
+    return Time.now() > expiryDate;
+  }
+
+  @Override
+  public UserGroupInformation verify(String user, String tokenStr)
+      throws SCMSecurityException {
+    if (conf.isGrpcBlockTokenEnabled()) {
+      if (Strings.isNullOrEmpty(tokenStr)) {
+        throw new BlockTokenException("Fail to find any token (empty or " +
+            "null.");
+      }
+      final Token<OzoneBlockTokenIdentifier> token = new Token();
+      OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier();
+      try {
+        token.decodeFromUrlString(tokenStr);
+        ByteArrayInputStream buf = new ByteArrayInputStream(
+            token.getIdentifier());
+        DataInputStream in = new DataInputStream(buf);
+        tokenId.readFields(in);
+      } catch (IOException ex) {
+        throw new BlockTokenException("Failed to decode token : " + tokenStr);
+      }
+
+      // TODO: revisit this when caClient is ready, skip signature check now.
+      /**
+       * the final code should like
+       * if (caClient == null) {
+       *   throw new SCMSecurityException("Certificate client not available to
+       *       validate token");
+       * }
+       */
+      if (caClient != null) {
+        X509Certificate singerCert = caClient.queryCertificate(
+            "certId=" + tokenId.getOmCertSerialId());
+        if (singerCert == null) {
+          throw new BlockTokenException("Can't find signer certificate " +
+              "(OmCertSerialId: " + tokenId.getOmCertSerialId() +
+              ") of the block token for user: " + tokenId.getUser());
+        }
+        Boolean validToken = caClient.verifySignature(tokenId.getBytes(),
+            token.getPassword(), singerCert);
+        if (!validToken) {
+          throw new BlockTokenException("Invalid block token for user: " +
+              tokenId.getUser());
+        }
+      }
+      // check expiration
+      if (isExpired(tokenId.getExpiryDate())) {
+        UserGroupInformation tokenUser = tokenId.getUser();
+        tokenUser.setAuthenticationMethod(
+            UserGroupInformation.AuthenticationMethod.TOKEN);
+        throw new BlockTokenException("Expired block token for user: " +
+            tokenUser);
+      }
+      // defer access mode, bcsid and maxLength check to container dispatcher
+      UserGroupInformation ugi = tokenId.getUser();
+      ugi.addToken(token);
+      ugi.setAuthenticationMethod(UserGroupInformation
+          .AuthenticationMethod.TOKEN);
+      return ugi;
+    } else {
+      return UserGroupInformation.createRemoteUser(user);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..89457fd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+  static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+  private long expiryDate;
+  private String ownerId;
+  private String blockId;
+  private EnumSet<AccessModeProto> modes;
+  private String omCertSerialId;
+  private long maxLength;
+
+  public OzoneBlockTokenIdentifier() {
+  }
+
+  public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+      EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId,
+      long maxLength) {
+    this.ownerId = ownerId;
+    this.blockId = blockId;
+    this.expiryDate = expiryDate;
+    this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+    this.omCertSerialId = omCertSerialId;
+    this.maxLength = maxLength;
+  }
+
+  @Override
+  public UserGroupInformation getUser() {
+    if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+      return UserGroupInformation.createRemoteUser(blockId);
+    }
+    return UserGroupInformation.createRemoteUser(ownerId);
+  }
+
+  public long getExpiryDate() {
+    return expiryDate;
+  }
+
+  public String getOwnerId() {
+    return ownerId;
+  }
+
+  public String getBlockId() {
+    return blockId;
+  }
+
+  public EnumSet<AccessModeProto> getAccessModes() {
+    return modes;
+  }
+
+  public String getOmCertSerialId(){
+    return omCertSerialId;
+  }
+
+  public long getMaxLength() {
+    return maxLength;
+  }
+
+  @Override
+  public Text getKind() {
+    return KIND_NAME;
+  }
+
+  @Override
+  public String toString() {
+    return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+        + ", ownerId=" + this.getOwnerId()
+        + ", omCertSerialId=" + this.getOmCertSerialId()
+        + ", blockId=" + this.getBlockId() + ", access modes="
+        + this.getAccessModes() + ", maxLength=" + this.getMaxLength() + ")";
+  }
+
+  static boolean isEqual(Object a, Object b) {
+    return a == null ? b == null : a.equals(b);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+
+    if (obj instanceof OzoneBlockTokenIdentifier) {
+      OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+      return new EqualsBuilder()
+          .append(this.expiryDate, that.expiryDate)
+          .append(this.ownerId, that.ownerId)
+          .append(this.blockId, that.blockId)
+          .append(this.modes, that.modes)
+          .append(this.omCertSerialId, that.omCertSerialId)
+          .append(this.maxLength, that.maxLength)
+          .build();
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(133, 567)
+        .append(this.expiryDate)
+        .append(this.blockId)
+        .append(this.ownerId)
+        .append(this.modes)
+        .append(this.omCertSerialId)
+        .append(this.maxLength)
+        .build();
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    final DataInputStream dis = (DataInputStream) in;
+    if (!dis.markSupported()) {
+      throw new IOException("Could not peek first byte.");
+    }
+    BlockTokenSecretProto tokenPtoto =
+        BlockTokenSecretProto.parseFrom((DataInputStream) in);
+    this.ownerId = tokenPtoto.getOwnerId();
+    this.blockId = tokenPtoto.getBlockId();
+    this.modes = EnumSet.copyOf(tokenPtoto.getModesList());
+    this.expiryDate = tokenPtoto.getExpiryDate();
+    this.omCertSerialId = tokenPtoto.getOmCertSerialId();
+    this.maxLength = tokenPtoto.getMaxLength();
+  }
+
+  @VisibleForTesting
+  public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+      throws IOException {
+    BlockTokenSecretProto tokenPtoto =
+        BlockTokenSecretProto.parseFrom((DataInputStream) in);
+    return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+        tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+        tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId(),
+        tokenPtoto.getMaxLength());
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    writeProtobuf(out);
+  }
+
+  @VisibleForTesting
+  void writeProtobuf(DataOutput out) throws IOException {
+    Builder builder = BlockTokenSecretProto.newBuilder()
+        .setBlockId(this.getBlockId())
+        .setOwnerId(this.getOwnerId())
+        .setOmCertSerialId(this.getOmCertSerialId())
+        .setExpiryDate(this.getExpiryDate())
+        .setMaxLength(this.getMaxLength());
+    // Add access mode allowed
+    for (AccessModeProto mode : this.getAccessModes()) {
+      builder.addModes(AccessModeProto.valueOf(mode.name()));
+    }
+    out.write(builder.build().toByteArray());
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..1943cce
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+    TokenSelector<OzoneBlockTokenIdentifier> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneBlockTokenSelector.class);
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+      Collection<Token<? extends TokenIdentifier>> tokens) {
+    if (service == null) {
+      return null;
+    }
+    for (Token<? extends TokenIdentifier> token : tokens) {
+      if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+        LOG.trace("Getting token for service:{}", service);
+        return (Token<OzoneBlockTokenIdentifier>) token;
+      }
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
new file mode 100644
index 0000000..d8170ab
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Ozone GRPC token header verifier.
+ */
+public interface TokenVerifier {
+  /**
+   * Given a user and tokenStr header, return a UGI object with token if
+   * verified.
+   * @param user user of the request
+   * @param tokenStr token str of the request
+   * @return UGI
+   * @throws SCMSecurityException
+   */
+  UserGroupInformation verify(String user, String tokenStr)
+      throws SCMSecurityException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..885bed5
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related test classes.
+ */
+package org.apache.hadoop.hdds.security.token;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 9231c8a..97627ca 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -34,6 +34,8 @@ import java.time.Duration;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
@@ -70,6 +72,7 @@ public class SecurityConfig {
   private final String publicKeyFileName;
   private final Duration certDuration;
   private final String x509SignatureAlgo;
+  private final Boolean grpcBlockTokenEnabled;
 
   /**
    * Constructs a SecurityConfig.
@@ -106,6 +109,10 @@ public class SecurityConfig {
     this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
         HDDS_X509_SIGNATURE_ALGO_DEFAULT);
 
+    this.grpcBlockTokenEnabled = this.configuration.getBoolean(
+        HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+        HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT);
+
     // First Startup -- if the provider is null, check for the provider.
     if (SecurityConfig.provider == null) {
       synchronized (SecurityConfig.class) {
@@ -213,6 +220,10 @@ public class SecurityConfig {
     return this.certDuration;
   }
 
+  public Boolean isGrpcBlockTokenEnabled() {
+    return this.grpcBlockTokenEnabled;
+  }
+
   /**
    * Adds a security provider dynamically if it is not loaded already.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index 9332e5b..5b5deef 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -19,8 +19,8 @@
 
 package org.apache.hadoop.hdds.security.x509.certificate.authority;
 
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.bouncycastle.cert.X509CertificateHolder;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index 1b6f576..bfc0576 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -91,6 +91,17 @@ public interface CertificateClient {
       X509Certificate cert);
 
   /**
+   * Verifies a digital Signature, given the signature and the certificate of
+   * the signer.
+   * @param data - Data in byte array.
+   * @param signature - Byte Array containing the signature.
+   * @param cert - Certificate of the Signer.
+   * @return true if verified, false if not.
+   */
+  boolean verifySignature(byte[] data, byte[] signature,
+      X509Certificate cert);
+
+  /**
    * Returns a CSR builder that can be used to creates a Certificate sigining
    * request.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
index 2e1f9df..0e762a5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
 import org.apache.logging.log4j.util.Strings;
 import org.bouncycastle.asn1.DEROctetString;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index f221246..85fba9b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -22,9 +22,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
 import org.apache.hadoop.util.Time;
 import org.apache.logging.log4j.util.Strings;
 import org.bouncycastle.asn1.x500.X500Name;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
index 4d2dcfd..5a8bf86 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -19,6 +19,8 @@
 
 package org.apache.hadoop.hdds.security.x509.exceptions;
 
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
 /**
  * Certificate Exceptions from the SCM Security layer.
  */
@@ -48,16 +50,4 @@ public class CertificateException extends SCMSecurityException {
   public CertificateException(Throwable cause) {
     super(cause);
   }
-
-  /**
-   * Ctor.
-   * @param message - Error Message
-   * @param cause  - Cause
-   * @param enableSuppression - Enable suppression.
-   * @param writableStackTrace - Writable stack trace.
-   */
-  public CertificateException(String message, Throwable cause,
-      boolean enableSuppression, boolean writableStackTrace) {
-    super(message, cause, enableSuppression, writableStackTrace);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
deleted file mode 100644
index e1f6296..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.exceptions;
-
-/**
- * Root Security Exception call for all Certificate related Execptions.
- */
-public class SCMSecurityException extends Exception {
-
-  /**
-   * Ctor.
-   * @param message - Error Message.
-   */
-  public SCMSecurityException(String message) {
-    super(message);
-  }
-
-  /**
-   * Ctor.
-   * @param message - Message.
-   * @param cause  - Actual cause.
-   */
-  public SCMSecurityException(String message, Throwable cause) {
-    super(message, cause);
-  }
-
-  /**
-   * Ctor.
-   * @param cause - Base Exception.
-   */
-  public SCMSecurityException(Throwable cause) {
-    super(cause);
-  }
-
-
-  /**
-   * Ctor.
-   * @param message - Error Message
-   * @param cause  - Cause
-   * @param enableSuppression - Enable suppression.
-   * @param writableStackTrace - Writable stack trace.
-   */
-  public SCMSecurityException(String message, Throwable cause,
-      boolean enableSuppression, boolean writableStackTrace) {
-    super(message, cause, enableSuppression, writableStackTrace);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index e7c354c..dad3bfe 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -19,6 +19,11 @@
 package org.apache.hadoop.ozone;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+
+import static org.apache.ratis.thirdparty.io.grpc.Metadata.ASCII_STRING_MARSHALLER;
 
 /**
  * Set of constants used in Ozone implementation.
@@ -231,4 +236,13 @@ public final class OzoneConsts {
   public static final String OM_METRICS_FILE = "omMetrics";
   public static final String OM_METRICS_TEMP_FILE = OM_METRICS_FILE + ".tmp";
 
+  // GRPC block token metadata header and context key
+  public static final String OZONE_BLOCK_TOKEN = "blocktoken";
+  public static final Context.Key<UserGroupInformation> UGI_CTX_KEY =
+      Context.key("UGI");
+
+  public static final Metadata.Key<String> OBT_METADATA_KEY =
+      Metadata.Key.of(OZONE_BLOCK_TOKEN, ASCII_STRING_MARSHALLER);
+  public static final Metadata.Key<String> USER_METADATA_KEY =
+      Metadata.Key.of(OZONE_USER, ASCII_STRING_MARSHALLER);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 8598fbf..53d11a5 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -210,7 +210,7 @@ message BlockTokenSecretProto {
     required uint64 expiryDate = 3;
     required string omCertSerialId = 4;
     repeated AccessModeProto modes = 5;
-
+    required uint64 maxLength = 6;
 }
 
 message BlockID {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..77a2cec
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestOzoneBlockTokenIdentifier.class);
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+  private static final String KEYSTORES_DIR =
+      new File(BASEDIR).getAbsolutePath();
+  private static long expiryTime;
+  private static KeyPair keyPair;
+  private static X509Certificate cert;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    File base = new File(BASEDIR);
+    FileUtil.fullyDelete(base);
+    base.mkdirs();
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    // Create Ozone Master certificate (SCM CA issued cert) and key store.
+    cert = KeyStoreTestUtil
+        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+  }
+
+  @After
+  public void cleanUp() throws Exception {
+    // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+  }
+
+  @Test
+  public void testSignToken() throws GeneralSecurityException, IOException {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+        .getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+        .getAbsolutePath();
+    String trustPassword = "trustPass";
+    String keyStorePassword = "keyStorePass";
+    String keyPassword = "keyPass";
+
+
+    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+        "OzoneMaster", keyPair.getPrivate(), cert);
+
+    // Create trust store and put the certificate in the trust store
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+        cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Sign the OzoneMaster Token with Ozone Master private key
+    PrivateKey privateKey = keyPair.getPrivate();
+    OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+        "testUser", "84940",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), 128L);
+    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+    // Verify a valid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+    // Verify an invalid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    tokenId = new OzoneBlockTokenIdentifier("", "",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), 128L);
+    LOG.info("Unsigned token {} is {}", tokenId,
+        verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+  }
+
+  @Test
+  public void testTokenSerialization() throws GeneralSecurityException,
+      IOException {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+        .getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+        .getAbsolutePath();
+    String trustPassword = "trustPass";
+    String keyStorePassword = "keyStorePass";
+    String keyPassword = "keyPass";
+    long maxLength = 128L;
+
+    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+        "OzoneMaster", keyPair.getPrivate(), cert);
+
+    // Create trust store and put the certificate in the trust store
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+        cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Sign the OzoneMaster Token with Ozone Master private key
+    PrivateKey privateKey = keyPair.getPrivate();
+    OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+        "testUser", "84940",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), maxLength);
+    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+
+    Token<BlockTokenIdentifier> token = new Token(tokenId.getBytes(),
+        signedToken, tokenId.getKind(), new Text("host:port"));
+
+    String encodeToUrlString = token.encodeToUrlString();
+
+    Token<BlockTokenIdentifier>decodedToken = new Token();
+    decodedToken.decodeFromUrlString(encodeToUrlString);
+
+    OzoneBlockTokenIdentifier decodedTokenId = new OzoneBlockTokenIdentifier();
+    decodedTokenId.readFields(new DataInputStream(
+        new ByteArrayInputStream(decodedToken.getIdentifier())));
+
+    Assert.assertEquals(decodedTokenId, tokenId);
+    Assert.assertEquals(decodedTokenId.getMaxLength(), maxLength);
+
+    // Verify a decoded signed Token with public key(certificate)
+    boolean isValidToken = verifyTokenAsymmetric(decodedTokenId, decodedToken
+        .getPassword(), cert);
+    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+  }
+
+
+  public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      PrivateKey privateKey) throws NoSuchAlgorithmException,
+      InvalidKeyException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initSign(privateKey);
+    rsaSignature.update(tokenId.getBytes());
+    byte[] signature = rsaSignature.sign();
+    return signature;
+  }
+
+  public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      byte[] signature, Certificate certificate) throws InvalidKeyException,
+      NoSuchAlgorithmException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initVerify(certificate);
+    rsaSignature.update(tokenId.getBytes());
+    boolean isValid = rsaSignature.verify(signature);
+    return isValid;
+  }
+
+  private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+      Mac mac, SecretKey key) {
+    try {
+      mac.init(key);
+    } catch (InvalidKeyException ike) {
+      throw new IllegalArgumentException("Invalid key to HMAC computation",
+          ike);
+    }
+    return mac.doFinal(identifier.getBytes());
+  }
+
+  OzoneBlockTokenIdentifier generateTestToken() {
+    return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+        RandomStringUtils.randomAlphabetic(5),
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString(), 1024768L);
+  }
+
+  @Test
+  public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+      CertificateEncodingException, NoSuchProviderException,
+      InvalidKeyException, SignatureException {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordAsym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+    // Create Ozone Master certificate (SCM CA issued cert) and key store
+    X509Certificate certificate;
+    certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+        kp, 30, "SHA256withRSA");
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordAsym.add(
+          signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+        duration / testTokenCount);
+
+    startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+          certificate);
+    }
+    duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+        + "is {} ns", duration / testTokenCount);
+  }
+
+  @Test
+  public void testSymmetricTokenPerf() {
+    String hmacSHA1 = "HmacSHA1";
+    String hmacSHA256 = "HmacSHA256";
+
+    testSymmetricTokenPerfHelper(hmacSHA1, 64);
+    testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+  }
+
+  public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordSym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyGenerator keyGen;
+    try {
+      keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+      keyGen.init(keyLen);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    Mac mac;
+    try {
+      mac = Mac.getInstance(hmacAlgorithm);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    SecretKey secretKey = keyGen.generateKey();
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordSym.add(
+          signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+        hmacAlgorithm, keyLen, duration / testTokenCount);
+  }
+
+  // TODO: verify certificate with a trust store
+  public boolean verifyCert(Certificate certificate) {
+    return true;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..d056655
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related classes.
+ */
+package org.apache.hadoop.hdds.security.token;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 0b9ef31..e8de1ea 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -18,8 +18,8 @@
 package org.apache.hadoop.hdds.security.x509.certificates;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
 import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
 import org.bouncycastle.asn1.ASN1Sequence;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 3a00ca0..16245a6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -20,8 +20,8 @@
 package org.apache.hadoop.hdds.security.x509.certificates;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
 import org.bouncycastle.asn1.x509.Extension;
 import org.bouncycastle.cert.X509CertificateHolder;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
new file mode 100644
index 0000000..968f0c8
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.transport.server;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.TokenVerifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Contexts;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.ServerCall;
+import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.Status;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.UGI_CTX_KEY;
+/**
+ * Grpc Server Interceptor for Ozone Block token.
+ */
+public class ServerCredentialInterceptor implements ServerInterceptor {
+
+
+  private static final ServerCall.Listener NOOP_LISTENER =
+      new ServerCall.Listener() {
+  };
+
+  private final TokenVerifier verifier;
+
+  ServerCredentialInterceptor(TokenVerifier verifier) {
+    this.verifier = verifier;
+  }
+
+  @Override
+  public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
+      ServerCall<ReqT, RespT> call, Metadata headers,
+      ServerCallHandler<ReqT, RespT> next) {
+    String token = headers.get(OBT_METADATA_KEY);
+    String user = headers.get(USER_METADATA_KEY);
+    Context ctx = Context.current();
+    try {
+      UserGroupInformation ugi = verifier.verify(user, token);
+      if (ugi == null) {
+        call.close(Status.UNAUTHENTICATED.withDescription("Missing Block " +
+            "Token from headers when block token is required."), headers);
+        return NOOP_LISTENER;
+      } else {
+        ctx = ctx.withValue(UGI_CTX_KEY, ugi);
+      }
+    } catch (SCMSecurityException e) {
+      call.close(Status.UNAUTHENTICATED.withDescription(e.getMessage())
+          .withCause(e), headers);
+      return NOOP_LISTENER;
+    }
+    return Contexts.interceptCall(ctx, call, headers, next);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4f7799d..2a3c99b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ozone.container.common.transport.server;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -30,6 +31,9 @@ import org.apache.hadoop.hdds.protocol.proto
 import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.common.helpers.
     StorageContainerException;
+import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -37,6 +41,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
 import org.apache.ratis.thirdparty.io.grpc.BindableService;
 import org.apache.ratis.thirdparty.io.grpc.Server;
 import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors;
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,8 +97,16 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
         DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
     NettyServerBuilder nettyServerBuilder =
         ((NettyServerBuilder) ServerBuilder.forPort(port))
-            .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
-            .addService(new GrpcXceiverService(dispatcher));
+            .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+
+    // Populate UGI context via ServerCredentialInterceptor
+    SecurityConfig secConfig = new SecurityConfig(conf);
+    ServerCredentialInterceptor credInterceptor =
+        new ServerCredentialInterceptor(
+            new BlockTokenVerifier(secConfig, getCaClient()));
+    nettyServerBuilder.addService(ServerInterceptors.intercept(
+          new GrpcXceiverService(dispatcher), credInterceptor));
+
     for (BindableService service : additionalServices) {
       nettyServerBuilder.addService(service);
     }
@@ -101,6 +114,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
     storageContainer = dispatcher;
   }
 
+  @VisibleForTesting
+  public CertificateClient getCaClient() {
+    // TODO: instantiate CertificateClient
+    return null;
+  }
+
   @Override
   public int getIPCPort() {
     return this.port;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
deleted file mode 100644
index 92b6d16..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.security;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.TokenIdentifier;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.EnumSet;
-
-/**
- * Block token identifier for Ozone/HDDS. Ozone block access token is similar
- * to HDFS block access token, which is meant to be lightweight and
- * short-lived. No need to renew or revoke a block access token. when a
- * cached block access token expires, the client simply get a new one.
- * Block access token should be cached only in memory and never write to disk.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenIdentifier extends TokenIdentifier {
-
-  static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
-  private long expiryDate;
-  private String ownerId;
-  private String blockId;
-  private final EnumSet<AccessModeProto> modes;
-  private final String omCertSerialId;
-
-  public OzoneBlockTokenIdentifier(String ownerId, String blockId,
-      EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
-    this.ownerId = ownerId;
-    this.blockId = blockId;
-    this.expiryDate = expiryDate;
-    this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
-    this.omCertSerialId = omCertSerialId;
-  }
-
-  @Override
-  public UserGroupInformation getUser() {
-    if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
-      return UserGroupInformation.createRemoteUser(blockId);
-    }
-    return UserGroupInformation.createRemoteUser(ownerId);
-  }
-
-  public long getExpiryDate() {
-    return expiryDate;
-  }
-
-  public String getOwnerId() {
-    return ownerId;
-  }
-
-  public String getBlockId() {
-    return blockId;
-  }
-
-  public EnumSet<AccessModeProto> getAccessModes() {
-    return modes;
-  }
-
-  public String getOmCertSerialId(){
-    return omCertSerialId;
-  }
-
-  @Override
-  public Text getKind() {
-    return KIND_NAME;
-  }
-
-  @Override
-  public String toString() {
-    return "block_token_identifier (expiryDate=" + this.getExpiryDate()
-        + ", ownerId=" + this.getOwnerId()
-        + ", omCertSerialId=" + this.getOmCertSerialId()
-        + ", blockId=" + this.getBlockId() + ", access modes="
-        + this.getAccessModes() + ")";
-  }
-
-  static boolean isEqual(Object a, Object b) {
-    return a == null ? b == null : a.equals(b);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-
-    if (obj instanceof OzoneBlockTokenIdentifier) {
-      OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
-      return new EqualsBuilder()
-          .append(this.expiryDate, that.expiryDate)
-          .append(this.ownerId, that.ownerId)
-          .append(this.blockId, that.blockId)
-          .append(this.modes, that.modes)
-          .append(this.omCertSerialId, that.omCertSerialId)
-          .build();
-    }
-    return false;
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(133, 567)
-        .append(this.expiryDate)
-        .append(this.blockId)
-        .append(this.ownerId)
-        .append(this.modes)
-        .append(this.omCertSerialId)
-        .build();
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    final DataInputStream dis = (DataInputStream) in;
-    if (!dis.markSupported()) {
-      throw new IOException("Could not peek first byte.");
-    }
-    readFieldsProtobuf(dis);
-  }
-
-  @VisibleForTesting
-  public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
-      throws IOException {
-    BlockTokenSecretProto tokenPtoto =
-        BlockTokenSecretProto.parseFrom((DataInputStream) in);
-    return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
-        tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
-        tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    writeProtobuf(out);
-  }
-
-  @VisibleForTesting
-  void writeProtobuf(DataOutput out) throws IOException {
-    Builder builder = BlockTokenSecretProto.newBuilder()
-        .setBlockId(this.getBlockId())
-        .setOwnerId(this.getOwnerId())
-        .setOmCertSerialId(this.getOmCertSerialId())
-        .setExpiryDate(this.getExpiryDate());
-    // Add access mode allowed
-    for (AccessModeProto mode : this.getAccessModes()) {
-      builder.addModes(AccessModeProto.valueOf(mode.name()));
-    }
-    out.write(builder.build().toByteArray());
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
deleted file mode 100644
index 086e19b..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-
-/**
- * A block token selector for Ozone.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenSelector implements
-    TokenSelector<OzoneBlockTokenIdentifier> {
-
-  private static final Logger LOG = LoggerFactory
-      .getLogger(OzoneBlockTokenSelector.class);
-
-  @Override
-  @SuppressWarnings("unchecked")
-  public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
-      Collection<Token<? extends TokenIdentifier>> tokens) {
-    if (service == null) {
-      return null;
-    }
-    for (Token<? extends TokenIdentifier> token : tokens) {
-      if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
-        LOG.trace("Getting token for service:{}", service);
-        return (Token<OzoneBlockTokenIdentifier>) token;
-      }
-    }
-    return null;
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[21/50] [abbrv] hadoop git commit: HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8af727f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8af727f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8af727f

Branch: refs/heads/HDDS-4
Commit: c8af727fa5efb58e9bfaeb2816d7d63c53e0bd62
Parents: 5c82193
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 22 13:32:28 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:50 2018 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  6 +-
 .../scm/protocol/ScmBlockLocationProtocol.java  |  2 +-
 .../StorageContainerLocationProtocol.java       |  3 +-
 .../protocolPB/ScmBlockLocationProtocolPB.java  |  4 +-
 .../StorageContainerLocationProtocolPB.java     |  2 +-
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |  1 -
 .../common/src/main/resources/ozone-default.xml | 31 +++++---
 .../StorageContainerDatanodeProtocol.java       |  2 +-
 .../StorageContainerDatanodeProtocolPB.java     |  2 +-
 .../scm/server/StorageContainerManager.java     | 18 ++---
 .../compose/compose-secure/docker-compose.yaml  |  6 +-
 .../test/compose/compose-secure/docker-config   | 12 +--
 .../acceptance/ozone-secure.robot               | 12 +--
 .../ozone/client/protocol/ClientProtocol.java   |  2 +-
 .../apache/hadoop/ozone/ksm/KSMConfigKeys.java  | 84 ++++++++++++++++++++
 .../ozone/om/protocol/OzoneManagerProtocol.java |  4 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    | 21 +++--
 .../apache/hadoop/ozone/om/OzoneManager.java    |  4 +-
 18 files changed, 151 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index e18fe91..376e6db 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -192,9 +192,9 @@ public final class ScmConfigKeys {
       "ozone.scm.http-address";
   public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
       "ozone.scm.https-address";
-  public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
-      "ozone.scm.kerberos.keytab.file";
-  public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
+  public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
+      "hdds.scm.kerberos.keytab.file";
+  public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal";
   public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
   public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
   public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index e17f1c2..2d46ae0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -33,7 +33,7 @@ import java.util.List;
  * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
  * to read/write a block.
  */
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ScmBlockLocationProtocol {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 5bc2521..e21bc53 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,7 +17,6 @@
 
 package org.apache.hadoop.hdds.scm.protocol;
 
-import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
  * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
  * that currently host a container.
  */
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface StorageContainerLocationProtocol {
   /**
    * Asks SCM where a container should be allocated. SCM responds with the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 89bb066..06bbd05 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,11 +18,9 @@
 package org.apache.hadoop.hdds.scm.protocolPB;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
     .ScmBlockLocationProtocolService;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.ipc.ProtocolInfo;
 import org.apache.hadoop.security.KerberosInfo;
 
@@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
     protocolVersion = 1)
 @InterfaceAudience.Private
 @KerberosInfo(
-    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ScmBlockLocationProtocolPB
     extends ScmBlockLocationProtocolService.BlockingInterface {
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index 3bd83f9..f80ba20 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
     "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
     protocolVersion = 1)
 @KerberosInfo(
-    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface StorageContainerLocationProtocolPB
     extends StorageContainerLocationProtocolService.BlockingInterface {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index b47113b..578a983 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -334,7 +334,6 @@ public final class OzoneConfigKeys {
 
   public static final String OZONE_CONTAINER_COPY_WORKDIR =
       "hdds.datanode.replication.work.dir";
-
   /**
    * Config properties to set client side checksum properties.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index d3e352b..13b47c9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -500,6 +500,7 @@
     </description>
   </property>
   <property>
+<<<<<<< HEAD
     <name>ozone.om.keytab.file</name>
     <value/>
     <tag>OM, SECURITY</tag>
@@ -509,6 +510,9 @@
   </property>
   <property>
     <name>ozone.om.db.cache.size.mb</name>
+=======
+    <name>ozone.ksm.db.cache.size.mb</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
     <value>128</value>
     <tag>OM, PERFORMANCE</tag>
     <description>
@@ -1019,7 +1023,7 @@
   </property>
 
   <property>
-    <name>ozone.scm.container.creation.lease.timeout</name>
+    <name>hdds.scm.container.creation.lease.timeout</name>
     <value>60s</value>
     <tag>OZONE, SCM</tag>
     <description>
@@ -1073,7 +1077,11 @@
     </description>
   </property>
   <property>
+<<<<<<< HEAD
     <name>hdds.container.close.threshold</name>
+=======
+    <name>hdds.scm.container.close.threshold</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
     <value>0.9f</value>
     <tag>OZONE, DATANODE</tag>
     <description>
@@ -1278,6 +1286,7 @@
     </description>
   </property>
   <property>
+<<<<<<< HEAD
     <name>hdds.db.profile</name>
     <value>DISK</value>
     <tag>OZONE, OM, PERFORMANCE</tag>
@@ -1388,16 +1397,16 @@
   </property>
 
   <property>
-    <name>ozone.scm.kerberos.keytab.file</name>
+    <name>hdds.scm.kerberos.keytab.file</name>
     <value></value>
     <tag> OZONE, SECURITY</tag>
     <description> The keytab file used by each SCM daemon to login as its
       service principal. The principal name is configured with
-      ozone.scm.kerberos.principal.
+      hdds.scm.kerberos.principal.
     </description>
   </property>
   <property>
-    <name>ozone.scm.kerberos.principal</name>
+    <name>hdds.scm.kerberos.principal</name>
     <value></value>
     <tag> OZONE, SECURITY</tag>
     <description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
@@ -1407,24 +1416,24 @@
     <name>ozone.om.kerberos.keytab.file</name>
     <value></value>
     <tag> OZONE, SECURITY</tag>
-    <description> The keytab file used by KSM daemon to login as its
+    <description> The keytab file used by OzoneManager daemon to login as its
       service principal. The principal name is configured with
-      hdds.ksm.kerberos.principal.
+      ozone.om.kerberos.principal.
     </description>
   </property>
   <property>
     <name>ozone.om.kerberos.principal</name>
     <value></value>
     <tag> OZONE, SECURITY</tag>
-    <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
+    <description>The OzoneManager service principal. Ex om/_HOST@REALM.COM</description>
   </property>
 
   <property>
-    <name>ozone.scm.web.authentication.kerberos.principal</name>
+    <name>hdds.scm.web.authentication.kerberos.principal</name>
     <value>HTTP/_HOST@EXAMPLE.COM</value>
   </property>
   <property>
-    <name>ozone.scm.web.authentication.kerberos.keytab</name>
+    <name>hdds.scm.web.authentication.kerberos.keytab</name>
     <value>/etc/security/keytabs/HTTP.keytab</value>
   </property>
 
@@ -1432,14 +1441,14 @@
     <name>ozone.om.http.kerberos.principal</name>
     <value>HTTP/_HOST@EXAMPLE.COM</value>
     <description>
-      KSM http server kerberos principal.
+      OzoneManager http server kerberos principal.
     </description>
   </property>
   <property>
     <name>ozone.om.http.kerberos.keytab.file</name>
     <value>/etc/security/keytabs/HTTP.keytab</value>
     <description>
-      KSM http server kerberos keytab.
+      OzoneManager http server kerberos keytab.
     </description>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index 8049e9d..3600581 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo;
  * Protoc file that defines this protocol.
  */
 @KerberosInfo(
-    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface StorageContainerDatanodeProtocol {
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9c32ef8..9006e91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
     "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
     protocolVersion = 1)
 @KerberosInfo(
-    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
     clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
 public interface StorageContainerDatanodeProtocolPB extends
     StorageContainerDatanodeProtocolService.BlockingInterface {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 519f51b..27fe326 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -108,8 +108,8 @@ import java.util.concurrent.TimeUnit;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
 /**
@@ -332,17 +332,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
   private void loginAsSCMUser(Configuration conf)
       throws IOException, AuthenticationException {
     LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
-            + "Principal: {}, keytab: {}", conf.get
-            (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
-        conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+            + "Principal: {}, keytab: {}",
+        conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY),
+        conf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY));
 
-    if (SecurityUtil.getAuthenticationMethod(conf).equals
-        (AuthenticationMethod.KERBEROS)) {
+    if (SecurityUtil.getAuthenticationMethod(conf).equals(
+        AuthenticationMethod.KERBEROS)) {
       UserGroupInformation.setConfiguration(conf);
       InetSocketAddress socAddr = HddsServerUtil
           .getScmBlockClientBindAddress(conf);
-      SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
-          OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+      SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+          HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
     } else {
       throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
           conf) + " authentication method not support. "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
index 2661163..db211bc 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -40,15 +40,15 @@ services:
       env_file:
         - ./docker-config
       command: ["/opt/hadoop/bin/ozone","datanode"]
-   ksm:
+   om:
       image: ahadoop/ozone:v1
-      hostname: ksm
+      hostname: om
       volumes:
          - ${OZONEDIR}:/opt/hadoop
       ports:
          - 9874:9874
       environment:
-         ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+         ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
       env_file:
           - ./docker-config
       command: ["/opt/hadoop/bin/ozone","ksm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 678c75a..360b69a 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.address=om
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
 OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
@@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm
 OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
 OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
 OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
 OZONE-SITE.XML_ozone.security.enabled=true
 OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
 OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.scm.client.address=scm
 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
@@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH
 
 OZONE_DATANODE_SECURE_USER=root
 CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYTABS=dn nn om scm HTTP testuser
 KERBEROS_KEYSTORES=hadoop
 KERBEROS_SERVER=ozone.kdc
 JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
index 4a78980..7fc1088 100644
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -26,7 +26,7 @@ ${version}
 *** Test Cases ***
 
 Daemons are running
-    Is daemon running           ksm
+    Is daemon running           om
     Is daemon running           scm
     Is daemon running           datanode
     Is daemon running           ozone.kdc
@@ -45,15 +45,15 @@ Test rest interface
                     Should contain      ${result}       200 OK
 
 Test ozone cli
-    ${result} =     Execute on      1   datanode        ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+    ${result} =     Execute on      1   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
                     Should contain      ${result}       Client cannot authenticate via
                     # Authenticate testuser
                     Execute on      0   datanode        kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
-                    Execute on      0   datanode        ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
-    ${result} =     Execute on      0   datanode        ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+                    Execute on      0   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
+    ${result} =     Execute on      0   datanode        ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
                     Should contain      ${result}       createdOn
-                    Execute on      0   datanode        ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
-    ${result} =     Execute on      0   datanode        ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+                    Execute on      0   datanode        ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
+    ${result} =     Execute on      0   datanode        ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
                     Should Be Equal     ${result}       bill
 
 *** Keywords ***

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index f5b2cd9..17634f2 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.security.KerberosInfo;
  * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
  * {@link  org.apache.hadoop.ozone.client.rest.RestClient} for REST.
  */
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ClientProtocol {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
new file mode 100644
index 0000000..cc25dbe
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+
+import org.apache.hadoop.ozone.OzoneAcl;
+/**
+ * KSM Constants.
+ */
+public final class KSMConfigKeys {
+  /**
+   * Never constructed.
+   */
+  private KSMConfigKeys() {
+  }
+
+
+  public static final String OZONE_KSM_HANDLER_COUNT_KEY =
+      "ozone.ksm.handler.count.key";
+  public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
+
+  public static final String OZONE_KSM_ADDRESS_KEY =
+      "ozone.ksm.address";
+  public static final String OZONE_KSM_BIND_HOST_DEFAULT =
+      "0.0.0.0";
+  public static final int OZONE_KSM_PORT_DEFAULT = 9862;
+
+  public static final String OZONE_KSM_HTTP_ENABLED_KEY =
+      "ozone.ksm.http.enabled";
+  public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
+      "ozone.ksm.http-bind-host";
+  public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
+      "ozone.ksm.https-bind-host";
+  public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
+      "ozone.ksm.http-address";
+  public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
+      "ozone.ksm.https-address";
+  public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
+  public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
+  public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
+
+  // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
+  public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
+      "ozone.ksm.db.cache.size.mb";
+  public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
+
+  public static final String OZONE_KSM_USER_MAX_VOLUME =
+      "ozone.ksm.user.max.volume";
+  public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
+
+  // KSM Default user/group permissions
+  public static final String OZONE_KSM_USER_RIGHTS =
+      "ozone.ksm.user.rights";
+  public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
+      OzoneAcl.OzoneACLRights.READ_WRITE;
+
+  public static final String OZONE_KSM_GROUP_RIGHTS =
+      "ozone.ksm.group.rights";
+  public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
+      OzoneAcl.OzoneACLRights.READ_WRITE;
+
+  public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
+      "ozone.key.deleting.limit.per.task";
+  public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
+
+  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
+      "ozone.om.web.authentication.kerberos.principal";
+  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+      "ozone.om.web.authentication.kerberos.keytab";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 2a4e864..233666a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -26,10 +26,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 
-
-import org.apache.hadoop.ozone.protocol.proto
-    .OzoneManagerProtocolProtos.OzoneAclInfo;
 import java.io.IOException;
 import java.util.List;
 import org.apache.hadoop.security.KerberosInfo;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5c96067..3fb5499 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -118,10 +118,10 @@ public final class TestSecureOzoneCluster {
   private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
       throws Exception {
     createPrincipal(scmKeytab,
-        conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
-    createPrincipal(spnegoKeytab,
-        conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
-        conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+        conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
+     createPrincipal(spnegoKeytab,
+         conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+         conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
     createPrincipal(omKeyTab,
         conf.get(OMConfigKeys
             .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
@@ -154,7 +154,7 @@ public final class TestSecureOzoneCluster {
         "kerberos");
     conf.set(OZONE_ADMINISTRATORS, curUser);
 
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
         "scm/" + host + "@" + realm);
     conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
         "HTTP_SCM/" + host + "@" + realm);
@@ -162,19 +162,18 @@ public final class TestSecureOzoneCluster {
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
         "om/" + host + "@" + realm);
     conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
-        "HTTP_KSM/" + host + "@" + realm);
+        "HTTP_OM/" + host + "@" + realm);
 
     scmKeytab = new File(workDir, "scm.keytab");
     spnegoKeytab = new File(workDir, "http.keytab");
     omKeyTab = new File(workDir, "om.keytab");
 
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
         scmKeytab.getAbsolutePath());
     conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
         spnegoKeytab.getAbsolutePath());
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
         omKeyTab.getAbsolutePath());
-
   }
 
   @Test
@@ -205,7 +204,7 @@ public final class TestSecureOzoneCluster {
   @Test
   public void testSecureScmStartupFailure() throws Exception {
     initSCM();
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
+    conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
 
@@ -215,9 +214,9 @@ public final class TestSecureOzoneCluster {
           StorageContainerManager.createSCM(null, conf);
         });
 
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
         "scm/_HOST@EXAMPLE.com");
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
         "/etc/security/keytabs/scm.keytab");
 
     testCommonKerberosFailures(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a05b2fd..7fc6aee 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -270,8 +270,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     if (SecurityUtil.getAuthenticationMethod(conf).equals
         (AuthenticationMethod.KERBEROS)) {
       LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
-              + "Principal: {},keytab: {}", conf.get
-              (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+              + "Principal: {},keytab: {}", conf.get(
+          OZONE_OM_KERBEROS_PRINCIPAL_KEY),
           conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
 
       UserGroupInformation.setConfiguration(conf);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[03/50] [abbrv] hadoop git commit: YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)

Posted by xy...@apache.org.
YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)

Change-Id: If6c7904aa17895e543cfca245264249eb7328bdc


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89764392
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89764392
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89764392

Branch: refs/heads/HDDS-4
Commit: 897643928c534062d45d00a36b95fd222299b4f6
Parents: 8ebeda9
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:39:06 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:39:06 2018 -0800

----------------------------------------------------------------------
 .../submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/89764392/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index b58ad77..2e84c96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -49,6 +49,7 @@ import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -218,7 +219,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
   private String generateCommandLaunchScript(RunJobParameters parameters,
       TaskType taskType, Component comp) throws IOException {
     File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
-    Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+    Writer w = new OutputStreamWriter(new FileOutputStream(file),
+        StandardCharsets.UTF_8);
     PrintWriter pw = new PrintWriter(w);
 
     try {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[48/50] [abbrv] hadoop git commit: HDDS-696. Bootstrap genesis SCM(CA) with self-signed certificate. Contributed by Anu Engineer.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 459dce7..ded50f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -28,7 +28,9 @@ import java.security.KeyPairGenerator;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 
-/** A class to generate Key Pair for use with Certificates. */
+/**
+ * A class to generate Key Pair for use with Certificates.
+ */
 public class HDDSKeyGenerator {
   private static final Logger LOG =
       LoggerFactory.getLogger(HDDSKeyGenerator.class);
@@ -44,7 +46,17 @@ public class HDDSKeyGenerator {
   }
 
   /**
+   * Constructor that takes a SecurityConfig as the Argument.
+   *
+   * @param config - SecurityConfig
+   */
+  public HDDSKeyGenerator(SecurityConfig config) {
+    this.securityConfig = config;
+  }
+
+  /**
    * Returns the Security config used for this object.
+   *
    * @return SecurityConfig
    */
   public SecurityConfig getSecurityConfig() {
@@ -55,10 +67,10 @@ public class HDDSKeyGenerator {
    * Use Config to generate key.
    *
    * @return KeyPair
-   * @throws NoSuchProviderException - On Error, due to missing Java
-   * dependencies.
+   * @throws NoSuchProviderException  - On Error, due to missing Java
+   *                                  dependencies.
    * @throws NoSuchAlgorithmException - On Error,  due to missing Java
-   * dependencies.
+   *                                  dependencies.
    */
   public KeyPair generateKey() throws NoSuchProviderException,
       NoSuchAlgorithmException {
@@ -71,10 +83,10 @@ public class HDDSKeyGenerator {
    *
    * @param size - int, valid key sizes.
    * @return KeyPair
-   * @throws NoSuchProviderException - On Error, due to missing Java
-   * dependencies.
+   * @throws NoSuchProviderException  - On Error, due to missing Java
+   *                                  dependencies.
    * @throws NoSuchAlgorithmException - On Error,  due to missing Java
-   * dependencies.
+   *                                  dependencies.
    */
   public KeyPair generateKey(int size) throws
       NoSuchProviderException, NoSuchAlgorithmException {
@@ -89,10 +101,10 @@ public class HDDSKeyGenerator {
    * @param algorithm - Algorithm to use
    * @param provider - Security provider.
    * @return KeyPair.
-   * @throws NoSuchProviderException - On Error, due to missing Java
-   * dependencies.
+   * @throws NoSuchProviderException  - On Error, due to missing Java
+   *                                  dependencies.
    * @throws NoSuchAlgorithmException - On Error,  due to missing Java
-   * dependencies.
+   *                                  dependencies.
    */
   public KeyPair generateKey(int size, String algorithm, String provider)
       throws NoSuchProviderException, NoSuchAlgorithmException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
deleted file mode 100644
index 95be1c4..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509.keys;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.output.FileWriterWithEncoding;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyPair;
-import java.util.Set;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * We store all Key material in good old PEM files.
- * This helps in avoiding dealing will persistent
- * Java KeyStore issues. Also when debugging,
- * general tools like OpenSSL can be used to read and
- * decode these files.
- */
-public class HDDSKeyPEMWriter {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
-  private final Path location;
-  private final SecurityConfig securityConfig;
-  private Set<PosixFilePermission> permissionSet =
-      Stream.of(OWNER_READ, OWNER_WRITE,  OWNER_EXECUTE)
-          .collect(Collectors.toSet());
-  private Supplier<Boolean> isPosixFileSystem;
-  public final static String PRIVATE_KEY = "PRIVATE KEY";
-  public final static String PUBLIC_KEY = "PUBLIC KEY";
-  public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
-  /*
-    Creates an HDDS Key Writer.
-
-    @param configuration - Configuration
-   */
-  public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
-    Preconditions.checkNotNull(configuration, "Config cannot be null");
-    this.securityConfig = new SecurityConfig(configuration);
-    isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
-    this.location = securityConfig.getKeyLocation();
-  }
-
-  /**
-   * Checks if File System supports posix style security permissions.
-   *
-   * @return True if it supports posix.
-   */
-  private static Boolean isPosix() {
-    return FileSystems.getDefault().supportedFileAttributeViews()
-        .contains("posix");
-  }
-
-  /**
-   * Returns the Permission set.
-   * @return Set
-   */
-  @VisibleForTesting
-  public Set<PosixFilePermission> getPermissionSet() {
-    return permissionSet;
-  }
-
-  /**
-   * Returns the Security config used for this object.
-   * @return SecurityConfig
-   */
-  public SecurityConfig getSecurityConfig() {
-    return securityConfig;
-  }
-
-  /**
-   * This function is used only for testing.
-   *
-   * @param isPosixFileSystem - Sets a boolean function for mimicking
-   * files systems that are not posix.
-   */
-  @VisibleForTesting
-  public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
-    this.isPosixFileSystem = isPosixFileSystem;
-  }
-
-  /**
-   * Writes a given key using the default config options.
-   *
-   * @param keyPair - Key Pair to write to file.
-   * @throws IOException - On I/O failure.
-   */
-  public void writeKey(KeyPair keyPair) throws IOException {
-    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
-        securityConfig.getPublicKeyFileName(), false);
-  }
-
-  /**
-   * Writes a given key using default config options.
-   *
-   * @param keyPair - Key pair to write
-   * @param overwrite - Overwrites the keys if they already exist.
-   * @throws IOException - On I/O failure.
-   */
-  public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
-    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
-        securityConfig.getPublicKeyFileName(), overwrite);
-  }
-
-  /**
-   * Writes a given key using default config options.
-   *
-   * @param basePath - The location to write to, override the config values.
-   * @param keyPair - Key pair to write
-   * @param overwrite - Overwrites the keys if they already exist.
-   * @throws IOException - On I/O failure.
-   */
-  public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
-      throws IOException {
-    writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
-        securityConfig.getPublicKeyFileName(), overwrite);
-  }
-
-  /**
-   * Helper function that actually writes data to the files.
-   *
-   * @param basePath - base path to write key
-   * @param keyPair - Key pair to write to file.
-   * @param privateKeyFileName - private key file name.
-   * @param publicKeyFileName - public key file name.
-   * @param force - forces overwriting the keys.
-   * @throws IOException - On I/O failure.
-   */
-  private synchronized void writeKey(Path basePath, KeyPair keyPair,
-      String privateKeyFileName, String publicKeyFileName, boolean force)
-      throws IOException {
-    checkPreconditions(basePath);
-
-    File privateKeyFile =
-        Paths.get(location.toString(), privateKeyFileName).toFile();
-    File publicKeyFile =
-        Paths.get(location.toString(), publicKeyFileName).toFile();
-    checkKeyFile(privateKeyFile, force, publicKeyFile);
-
-    try (PemWriter privateKeyWriter = new PemWriter(new
-        FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
-      privateKeyWriter.writeObject(
-          new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
-    }
-
-    try (PemWriter publicKeyWriter = new PemWriter(new
-        FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
-      publicKeyWriter.writeObject(
-          new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
-    }
-    Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
-    Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
-  }
-
-  /**
-   * Checks if private and public key file already exists. Throws IOException
-   * if file exists and force flag is set to false, else will delete the
-   * existing file.
-   *
-   * @param privateKeyFile - Private key file.
-   * @param force - forces overwriting the keys.
-   * @param publicKeyFile - public key file.
-   * @throws IOException - On I/O failure.
-   */
-  private void checkKeyFile(File privateKeyFile, boolean force,
-      File publicKeyFile) throws IOException {
-    if (privateKeyFile.exists() && force) {
-      if (!privateKeyFile.delete()) {
-        throw new IOException("Unable to delete private key file.");
-      }
-    }
-
-    if (publicKeyFile.exists() && force) {
-      if (!publicKeyFile.delete()) {
-        throw new IOException("Unable to delete public key file.");
-      }
-    }
-
-    if (privateKeyFile.exists()) {
-      throw new IOException("Private Key file already exists.");
-    }
-
-    if (publicKeyFile.exists()) {
-      throw new IOException("Public Key file already exists.");
-    }
-  }
-
-  /**
-   * Checks if base path exists and sets file permissions.
-   *
-   * @param basePath - base path to write key
-   * @throws IOException - On I/O failure.
-   */
-  private void checkPreconditions(Path basePath) throws IOException {
-    Preconditions.checkNotNull(basePath, "Base path cannot be null");
-    if (!isPosixFileSystem.get()) {
-      LOG.error("Keys cannot be stored securely without POSIX file system "
-          + "support for now.");
-      throw new IOException("Unsupported File System for pem file.");
-    }
-
-    if (Files.exists(basePath)) {
-      // Not the end of the world if we reset the permissions on an existing
-      // directory.
-      Files.setPosixFilePermissions(basePath, permissionSet);
-    } else {
-      boolean success = basePath.toFile().mkdirs();
-      if (!success) {
-        LOG.error("Unable to create the directory for the "
-            + "location. Location: {}", basePath);
-        throw new IOException("Unable to create the directory for the "
-            + "location. Location:" + basePath);
-      }
-      Files.setPosixFilePermissions(basePath, permissionSet);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
new file mode 100644
index 0000000..1d45ef1
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files. This helps in avoiding
+ * dealing will persistent Java KeyStore issues. Also when debugging, general
+ * tools like OpenSSL can be used to read and decode these files.
+ */
+public class KeyCodec {
+  public final static String PRIVATE_KEY = "PRIVATE KEY";
+  public final static String PUBLIC_KEY = "PUBLIC KEY";
+  public final static Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+  private final static  Logger LOG =
+      LoggerFactory.getLogger(KeyCodec.class);
+  private final Path location;
+  private final SecurityConfig securityConfig;
+  private Set<PosixFilePermission> permissionSet =
+      Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+          .collect(Collectors.toSet());
+  private Supplier<Boolean> isPosixFileSystem;
+
+  /**
+   * Creates an KeyCodec.
+   *
+   * @param config - Security Config.
+   * @param component - Component String.
+   */
+  public KeyCodec(SecurityConfig config, String component) {
+    this.securityConfig = config;
+    isPosixFileSystem = KeyCodec::isPosix;
+    this.location = securityConfig.getKeyLocation(component);
+  }
+
+  /**
+   * Creates an HDDS Key Writer.
+   *
+   * @param configuration - Configuration
+   */
+  public KeyCodec(Configuration configuration) {
+    Preconditions.checkNotNull(configuration, "Config cannot be null");
+    this.securityConfig = new SecurityConfig(configuration);
+    isPosixFileSystem = KeyCodec::isPosix;
+    this.location = securityConfig.getKeyLocation();
+  }
+
+  /**
+   * Checks if File System supports posix style security permissions.
+   *
+   * @return True if it supports posix.
+   */
+  private static Boolean isPosix() {
+    return FileSystems.getDefault().supportedFileAttributeViews()
+        .contains("posix");
+  }
+
+  /**
+   * Returns the Permission set.
+   *
+   * @return Set
+   */
+  @VisibleForTesting
+  public Set<PosixFilePermission> getPermissionSet() {
+    return permissionSet;
+  }
+
+  /**
+   * Returns the Security config used for this object.
+   *
+   * @return SecurityConfig
+   */
+  public SecurityConfig getSecurityConfig() {
+    return securityConfig;
+  }
+
+  /**
+   * This function is used only for testing.
+   *
+   * @param isPosixFileSystem - Sets a boolean function for mimicking files
+   * systems that are not posix.
+   */
+  @VisibleForTesting
+  public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+    this.isPosixFileSystem = isPosixFileSystem;
+  }
+
+  /**
+   * Writes a given key using the default config options.
+   *
+   * @param keyPair - Key Pair to write to file.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(KeyPair keyPair) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), false);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), overwrite);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param basePath - The location to write to, override the config values.
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+      throws IOException {
+    writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), overwrite);
+  }
+
+  /**
+   * Reads a Private Key from the PEM Encoded Store.
+   *
+   * @param basePath - Base Path, Directory where the Key is stored.
+   * @param keyFileName - File Name of the private key
+   * @return PrivateKey Object.
+   * @throws IOException - on Error.
+   */
+  private PKCS8EncodedKeySpec readKey(Path basePath, String keyFileName)
+      throws IOException {
+    File fileName = Paths.get(basePath.toString(), keyFileName).toFile();
+    String keyData = FileUtils.readFileToString(fileName, DEFAULT_CHARSET);
+    final byte[] pemContent;
+    try (PemReader pemReader = new PemReader(new StringReader(keyData))) {
+      PemObject keyObject = pemReader.readPemObject();
+      pemContent = keyObject.getContent();
+    }
+    return new PKCS8EncodedKeySpec(pemContent);
+  }
+
+  /**
+   * Returns a Private Key from a PEM encoded file.
+   *
+   * @param basePath - base path
+   * @param privateKeyFileName - private key file name.
+   * @return PrivateKey
+   * @throws InvalidKeySpecException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws IOException              - on Error.
+   */
+  public PrivateKey readPrivateKey(Path basePath, String privateKeyFileName)
+      throws InvalidKeySpecException, NoSuchAlgorithmException, IOException {
+    PKCS8EncodedKeySpec encodedKeySpec = readKey(basePath, privateKeyFileName);
+    final KeyFactory keyFactory =
+        KeyFactory.getInstance(securityConfig.getProvider());
+    final PrivateKey privateKey =
+        keyFactory.generatePrivate(encodedKeySpec);
+    return privateKey;
+  }
+
+  /**
+   * Returns a public key from a PEM encoded file.
+   *
+   * @param basePath - base path.
+   * @param publicKeyFileName - public key file name.
+   * @return PublicKey
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws InvalidKeySpecException  - on Error.
+   * @throws IOException              - on Error.
+   */
+  public PublicKey readPublicKey(Path basePath, String publicKeyFileName)
+      throws NoSuchAlgorithmException, InvalidKeySpecException, IOException {
+    PKCS8EncodedKeySpec encodedKeySpec = readKey(basePath, publicKeyFileName);
+    final KeyFactory keyFactory =
+        KeyFactory.getInstance(securityConfig.getProvider());
+    final PublicKey publicKey =
+        keyFactory.generatePublic(encodedKeySpec);
+    return publicKey;
+  }
+
+  /**
+   * Helper function that actually writes data to the files.
+   *
+   * @param basePath - base path to write key
+   * @param keyPair - Key pair to write to file.
+   * @param privateKeyFileName - private key file name.
+   * @param publicKeyFileName - public key file name.
+   * @param force - forces overwriting the keys.
+   * @throws IOException - On I/O failure.
+   */
+  private synchronized void writeKey(Path basePath, KeyPair keyPair,
+      String privateKeyFileName, String publicKeyFileName, boolean force)
+      throws IOException {
+    checkPreconditions(basePath);
+
+    File privateKeyFile =
+        Paths.get(location.toString(), privateKeyFileName).toFile();
+    File publicKeyFile =
+        Paths.get(location.toString(), publicKeyFileName).toFile();
+    checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+    try (PemWriter privateKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+      privateKeyWriter.writeObject(
+          new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+    }
+
+    try (PemWriter publicKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+      publicKeyWriter.writeObject(
+          new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+    }
+    Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+    Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+  }
+
+  /**
+   * Checks if private and public key file already exists. Throws IOException if
+   * file exists and force flag is set to false, else will delete the existing
+   * file.
+   *
+   * @param privateKeyFile - Private key file.
+   * @param force - forces overwriting the keys.
+   * @param publicKeyFile - public key file.
+   * @throws IOException - On I/O failure.
+   */
+  private void checkKeyFile(File privateKeyFile, boolean force,
+      File publicKeyFile) throws IOException {
+    if (privateKeyFile.exists() && force) {
+      if (!privateKeyFile.delete()) {
+        throw new IOException("Unable to delete private key file.");
+      }
+    }
+
+    if (publicKeyFile.exists() && force) {
+      if (!publicKeyFile.delete()) {
+        throw new IOException("Unable to delete public key file.");
+      }
+    }
+
+    if (privateKeyFile.exists()) {
+      throw new IOException("Private Key file already exists.");
+    }
+
+    if (publicKeyFile.exists()) {
+      throw new IOException("Public Key file already exists.");
+    }
+  }
+
+  /**
+   * Checks if base path exists and sets file permissions.
+   *
+   * @param basePath - base path to write key
+   * @throws IOException - On I/O failure.
+   */
+  private void checkPreconditions(Path basePath) throws IOException {
+    Preconditions.checkNotNull(basePath, "Base path cannot be null");
+    if (!isPosixFileSystem.get()) {
+      LOG.error("Keys cannot be stored securely without POSIX file system "
+          + "support for now.");
+      throw new IOException("Unsupported File System for pem file.");
+    }
+
+    if (Files.exists(basePath)) {
+      // Not the end of the world if we reset the permissions on an existing
+      // directory.
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    } else {
+      boolean success = basePath.toFile().mkdirs();
+      if (!success) {
+        LOG.error("Unable to create the directory for the "
+            + "location. Location: {}", basePath);
+        throw new IOException("Unable to create the directory for the "
+            + "location. Location:" + basePath);
+      }
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
index 1cf39b2..ad26f77 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
@@ -62,7 +62,7 @@ public class StorageInfo {
    * @param cT
    *          Cluster creation Time
 
-   * @throws IOException
+   * @throws IOException - on Error.
    */
   public StorageInfo(NodeType type, String cid, long cT)
       throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
new file mode 100644
index 0000000..0e98ba7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.cert.CertificateException;
+import java.util.function.Consumer;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.junit.Assert.*;
+
+/**
+ * Tests the Default CA Server.
+ */
+public class TestDefaultCAServer {
+  private static OzoneConfiguration conf = new OzoneConfiguration();
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Before
+  public void init() throws IOException {
+    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+  }
+
+  @Test
+  public void testInit() throws SCMSecurityException, CertificateException,
+      IOException {
+    SecurityConfig securityConfig = new SecurityConfig(conf);
+    CertificateServer testCA = new DefaultCAServer("testCA",
+        RandomStringUtils.randomAlphabetic(4),
+        RandomStringUtils.randomAlphabetic(4));
+    testCA.init(securityConfig, CertificateServer.CAType.SELF_SIGNED_CA);
+    X509CertificateHolder first = testCA.getCACertificate();
+    assertNotNull(first);
+    //Init is idempotent.
+    testCA.init(securityConfig, CertificateServer.CAType.SELF_SIGNED_CA);
+    X509CertificateHolder second = testCA.getCACertificate();
+    assertEquals(first, second);
+
+    // we only support Self Signed CA for now.
+    try {
+      testCA.init(securityConfig, CertificateServer.CAType.INTERMEDIARY_CA);
+      fail("code should not reach here, exception should have been thrown.");
+    } catch (IllegalStateException e) {
+      // This is a run time exception, hence it is not caught by the junit
+      // expected Exception.
+      assertTrue(e.toString().contains("Not implemented"));
+    }
+  }
+
+  @Test
+  public void testMissingCertificate() {
+    SecurityConfig securityConfig = new SecurityConfig(conf);
+    CertificateServer testCA = new DefaultCAServer("testCA",
+        RandomStringUtils.randomAlphabetic(4),
+        RandomStringUtils.randomAlphabetic(4));
+    Consumer<SecurityConfig> caInitializer =
+        ((DefaultCAServer) testCA).processVerificationStatus(
+        DefaultCAServer.VerificationStatus.MISSING_CERTIFICATE);
+    try {
+
+      caInitializer.accept(securityConfig);
+      fail("code should not reach here, exception should have been thrown.");
+    } catch (IllegalStateException e) {
+      // This also is a runtime exception. Hence not caught by junit expected
+      // exception.
+      assertTrue(e.toString().contains("Missing Root Certs"));
+    }
+  }
+
+  @Test
+  public void testMissingKey() {
+    SecurityConfig securityConfig = new SecurityConfig(conf);
+    CertificateServer testCA = new DefaultCAServer("testCA",
+        RandomStringUtils.randomAlphabetic(4),
+        RandomStringUtils.randomAlphabetic(4));
+    Consumer<SecurityConfig> caInitializer =
+        ((DefaultCAServer) testCA).processVerificationStatus(
+            DefaultCAServer.VerificationStatus.MISSING_KEYS);
+    try {
+
+      caInitializer.accept(securityConfig);
+      fail("code should not reach here, exception should have been thrown.");
+    } catch (IllegalStateException e) {
+      // This also is a runtime exception. Hence not caught by junit expected
+      // exception.
+      assertTrue(e.toString().contains("Missing Keys"));
+    }
+
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
new file mode 100644
index 0000000..1d20a78
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Tests for Default CA.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
new file mode 100644
index 0000000..9ac956f
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the Certificate codecs.
+ */
+public class TestCertificateCodec {
+  private static OzoneConfiguration conf = new OzoneConfiguration();
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Before
+  public void init() throws IOException {
+    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+  }
+
+  /**
+   * This test converts a X509Certificate Holder object to a PEM encoded String,
+   * then creates a new X509Certificate object to verify that we are able to
+   * serialize and deserialize correctly. we follow up with converting these
+   * objects to standard JCA x509Certificate objects.
+   *
+   * @throws NoSuchProviderException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws IOException              - on Error.
+   * @throws SCMSecurityException     - on Error.
+   * @throws CertificateException     - on Error.
+   */
+  @Test
+  public void testGetPEMEncodedString()
+      throws NoSuchProviderException, NoSuchAlgorithmException,
+      IOException, SCMSecurityException, CertificateException {
+    HDDSKeyGenerator keyGenerator =
+        new HDDSKeyGenerator(conf);
+    X509CertificateHolder cert =
+        SelfSignedCertificate.newBuilder()
+            .setSubject(RandomStringUtils.randomAlphabetic(4))
+            .setClusterID(RandomStringUtils.randomAlphabetic(4))
+            .setScmID(RandomStringUtils.randomAlphabetic(4))
+            .setBeginDate(LocalDate.now())
+            .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+            .setConfiguration(keyGenerator.getSecurityConfig()
+                .getConfiguration())
+            .setKey(keyGenerator.generateKey())
+            .makeCA()
+            .build();
+    CertificateCodec codec = new CertificateCodec(conf);
+    String pemString = codec.getPEMEncodedString(cert);
+    assertTrue(pemString.startsWith(CertificateCodec.BEGIN_CERT));
+    assertTrue(pemString.endsWith(CertificateCodec.END_CERT + "\n"));
+
+    // Read back the certificate and verify that all the comparisons pass.
+    X509CertificateHolder newCert =
+        codec.getCertificateHolder(codec.getX509Certificate(pemString));
+    assertEquals(cert, newCert);
+
+    // Just make sure we can decode both these classes to Java Std. lIb classes.
+    X509Certificate firstCert = CertificateCodec.getX509Certificate(cert);
+    X509Certificate secondCert = CertificateCodec.getX509Certificate(newCert);
+    assertEquals(firstCert, secondCert);
+  }
+
+  /**
+   * tests writing and reading certificates in PEM encoded form.
+   *
+   * @throws NoSuchProviderException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws IOException              - on Error.
+   * @throws SCMSecurityException     - on Error.
+   * @throws CertificateException     - on Error.
+   */
+  @Test
+  public void testwriteCertificate() throws NoSuchProviderException,
+      NoSuchAlgorithmException, IOException, SCMSecurityException,
+      CertificateException {
+    HDDSKeyGenerator keyGenerator =
+        new HDDSKeyGenerator(conf);
+    X509CertificateHolder cert =
+        SelfSignedCertificate.newBuilder()
+            .setSubject(RandomStringUtils.randomAlphabetic(4))
+            .setClusterID(RandomStringUtils.randomAlphabetic(4))
+            .setScmID(RandomStringUtils.randomAlphabetic(4))
+            .setBeginDate(LocalDate.now())
+            .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+            .setConfiguration(keyGenerator.getSecurityConfig()
+                .getConfiguration())
+            .setKey(keyGenerator.generateKey())
+            .makeCA()
+            .build();
+    CertificateCodec codec = new CertificateCodec(conf);
+    String pemString = codec.getPEMEncodedString(cert);
+    File basePath = temporaryFolder.newFolder();
+    if (!basePath.exists()) {
+      Assert.assertTrue(basePath.mkdirs());
+    }
+    codec.writeCertificate(basePath.toPath(), "pemcertificate.crt",
+        pemString, false);
+    X509CertificateHolder certHolder =
+        codec.readCertificate(basePath.toPath(), "pemcertificate.crt");
+    assertNotNull(certHolder);
+    assertEquals(cert.getSerialNumber(), certHolder.getSerialNumber());
+  }
+
+  /**
+   * Tests reading and writing certificates in DER form.
+   *
+   * @throws IOException              - on Error.
+   * @throws SCMSecurityException     - on Error.
+   * @throws CertificateException     - on Error.
+   * @throws NoSuchProviderException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   */
+  @Test
+  public void testwriteCertificateDefault()
+      throws IOException, SCMSecurityException, CertificateException,
+      NoSuchProviderException, NoSuchAlgorithmException {
+    HDDSKeyGenerator keyGenerator =
+        new HDDSKeyGenerator(conf);
+    X509CertificateHolder cert =
+        SelfSignedCertificate.newBuilder()
+            .setSubject(RandomStringUtils.randomAlphabetic(4))
+            .setClusterID(RandomStringUtils.randomAlphabetic(4))
+            .setScmID(RandomStringUtils.randomAlphabetic(4))
+            .setBeginDate(LocalDate.now())
+            .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+            .setConfiguration(keyGenerator.getSecurityConfig()
+                .getConfiguration())
+            .setKey(keyGenerator.generateKey())
+            .makeCA()
+            .build();
+    CertificateCodec codec = new CertificateCodec(conf);
+    codec.writeCertificate(cert);
+    X509CertificateHolder certHolder = codec.readCertificate();
+    assertNotNull(certHolder);
+    assertEquals(cert.getSerialNumber(), certHolder.getSerialNumber());
+  }
+
+  /**
+   * Tests writing to non-default certificate file name.
+   *
+   * @throws IOException              - on Error.
+   * @throws SCMSecurityException     - on Error.
+   * @throws NoSuchProviderException  - on Error.
+   * @throws NoSuchAlgorithmException - on Error.
+   * @throws CertificateException     - on Error.
+   */
+  @Test
+  public void writeCertificate2() throws IOException, SCMSecurityException,
+      NoSuchProviderException, NoSuchAlgorithmException, CertificateException {
+    HDDSKeyGenerator keyGenerator =
+        new HDDSKeyGenerator(conf);
+    X509CertificateHolder cert =
+        SelfSignedCertificate.newBuilder()
+            .setSubject(RandomStringUtils.randomAlphabetic(4))
+            .setClusterID(RandomStringUtils.randomAlphabetic(4))
+            .setScmID(RandomStringUtils.randomAlphabetic(4))
+            .setBeginDate(LocalDate.now())
+            .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+            .setConfiguration(keyGenerator.getSecurityConfig()
+                .getConfiguration())
+            .setKey(keyGenerator.generateKey())
+            .makeCA()
+            .build();
+    CertificateCodec codec =
+        new CertificateCodec(keyGenerator.getSecurityConfig(), "ca");
+    codec.writeCertificate(cert, "newcert.crt", false);
+    // Rewrite with force support
+    codec.writeCertificate(cert, "newcert.crt", true);
+    X509CertificateHolder x509CertificateHolder =
+        codec.readCertificate(codec.getLocation(), "newcert.crt");
+    assertNotNull(x509CertificateHolder);
+
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 9328c50..d234b66 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,20 +6,22 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
-package org.apache.hadoop.hdds.security.x509.certificates;
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
 import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
 import org.bouncycastle.asn1.ASN1Sequence;
@@ -45,12 +47,15 @@ import java.util.UUID;
 
 import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 
+/**
+ * Certificate Signing Request.
+ */
 public class TestCertificateSignRequest {
 
-  private SecurityConfig securityConfig;
   private static OzoneConfiguration conf = new OzoneConfiguration();
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private SecurityConfig securityConfig;
 
   @Before
   public void init() throws IOException {
@@ -257,7 +262,7 @@ public class TestCertificateSignRequest {
     // Verify CSR with attribute for extensions
     Assert.assertEquals(1, csr.getAttributes().length);
   }
-  
+
   @Test
   public void testCsrSerialization() throws NoSuchProviderException,
       NoSuchAlgorithmException, SCMSecurityException, IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index e581dc8..416cc61 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -5,7 +5,7 @@
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
+ * with the License.  You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -17,11 +17,12 @@
  *
  */
 
-package org.apache.hadoop.hdds.security.x509.certificates;
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
 import org.bouncycastle.asn1.x509.Extension;
 import org.bouncycastle.cert.X509CertificateHolder;
@@ -41,8 +42,8 @@ import java.security.NoSuchProviderException;
 import java.security.SignatureException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.time.Duration;
-import java.time.Instant;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
 import java.util.Date;
 import java.util.UUID;
 
@@ -52,10 +53,10 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
  * Test Class for Root Certificate generation.
  */
 public class TestRootCertificate {
-  private SecurityConfig securityConfig;
   private static OzoneConfiguration conf = new OzoneConfiguration();
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private SecurityConfig securityConfig;
 
   @Before
   public void init() throws IOException {
@@ -67,10 +68,9 @@ public class TestRootCertificate {
   public void testAllFieldsAreExpected()
       throws SCMSecurityException, NoSuchProviderException,
       NoSuchAlgorithmException, CertificateException,
-      SignatureException, InvalidKeyException {
-    Instant now = Instant.now();
-    Date notBefore = Date.from(now);
-    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+      SignatureException, InvalidKeyException, IOException {
+    LocalDate notBefore = LocalDate.now();
+    LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
     String clusterID = UUID.randomUUID().toString();
     String scmID = UUID.randomUUID().toString();
     String subject = "testRootCert";
@@ -96,13 +96,15 @@ public class TestRootCertificate {
 
 
     // Make sure that NotBefore is before the current Date
-    Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+    Date invalidDate = java.sql.Date.valueOf(
+        notBefore.minus(1, ChronoUnit.DAYS));
     Assert.assertFalse(
         certificateHolder.getNotBefore()
             .before(invalidDate));
 
     //Make sure the end date is honored.
-    invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+    invalidDate = java.sql.Date.valueOf(
+        notAfter.plus(1, ChronoUnit.DAYS));
     Assert.assertFalse(
         certificateHolder.getNotAfter()
             .after(invalidDate));
@@ -113,7 +115,8 @@ public class TestRootCertificate {
     Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
     Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
 
-    // We did not ask for this Certificate to be a CA certificate, hence that
+    // We did not ask for this Certificate to be a CertificateServer
+    // certificate, hence that
     // extension should be null.
     Assert.assertNull(
         certificateHolder.getExtension(Extension.basicConstraints));
@@ -128,10 +131,9 @@ public class TestRootCertificate {
   @Test
   public void testCACert()
       throws SCMSecurityException, NoSuchProviderException,
-      NoSuchAlgorithmException {
-    Instant now = Instant.now();
-    Date notBefore = Date.from(now);
-    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+      NoSuchAlgorithmException, IOException {
+    LocalDate notBefore = LocalDate.now();
+    LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
     String clusterID = UUID.randomUUID().toString();
     String scmID = UUID.randomUUID().toString();
     String subject = "testRootCert";
@@ -151,7 +153,8 @@ public class TestRootCertificate {
             .makeCA();
 
     X509CertificateHolder certificateHolder = builder.build();
-    // This time we asked for a CA Certificate, make sure that extension is
+    // This time we asked for a CertificateServer Certificate, make sure that
+    // extension is
     // present and valid.
     Extension basicExt =
         certificateHolder.getExtension(Extension.basicConstraints);
@@ -167,10 +170,9 @@ public class TestRootCertificate {
   @Test
   public void testInvalidParamFails()
       throws SCMSecurityException, NoSuchProviderException,
-      NoSuchAlgorithmException {
-    Instant now = Instant.now();
-    Date notBefore = Date.from(now);
-    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+      NoSuchAlgorithmException, IOException {
+    LocalDate notBefore = LocalDate.now();
+    LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
     String clusterID = UUID.randomUUID().toString();
     String scmID = UUID.randomUUID().toString();
     String subject = "testRootCert";
@@ -253,6 +255,4 @@ public class TestRootCertificate {
     // Assert that we can create a certificate with all sane params.
     Assert.assertNotNull(builder.build());
   }
-
-
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
index c8a31fc..fffe1e5 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -5,7 +5,7 @@
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
+ * with the License.  You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
@@ -19,4 +19,4 @@
 /**
  * Test classes for Certificate utilities.
  */
-package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
deleted file mode 100644
index db5d430..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.keys;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test class for HDDS pem writer.
- */
-public class TestHDDSKeyPEMWriter {
-
-  @Rule
-  public TemporaryFolder temporaryFolder = new TemporaryFolder();
-  private OzoneConfiguration configuration;
-  private HDDSKeyGenerator keyGenerator;
-  private String prefix;
-
-  @Before
-  public void init() throws IOException {
-    configuration = new OzoneConfiguration();
-    prefix = temporaryFolder.newFolder().toString();
-    configuration.set(HDDS_METADATA_DIR_NAME, prefix);
-    keyGenerator = new HDDSKeyGenerator(configuration);
-  }
-
-  /**
-   * Assert basic things like we are able to create a file, and the names are
-   * in expected format etc.
-   *
-   * @throws NoSuchProviderException - On Error, due to missing Java
-   * dependencies.
-   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
-   * dependencies.
-   * @throws IOException - On I/O failure.
-   */
-  @Test
-  public void testWriteKey()
-      throws NoSuchProviderException, NoSuchAlgorithmException,
-      IOException, InvalidKeySpecException {
-    KeyPair keys = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    pemWriter.writeKey(keys);
-
-    // Assert that locations have been created.
-    Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
-    Assert.assertTrue(keyLocation.toFile().exists());
-
-    // Assert that locations are created in the locations that we specified
-    // using the Config.
-    Assert.assertTrue(keyLocation.toString().startsWith(prefix));
-    Path privateKeyPath = Paths.get(keyLocation.toString(),
-        pemWriter.getSecurityConfig().getPrivateKeyFileName());
-    Assert.assertTrue(privateKeyPath.toFile().exists());
-    Path publicKeyPath = Paths.get(keyLocation.toString(),
-        pemWriter.getSecurityConfig().getPublicKeyFileName());
-    Assert.assertTrue(publicKeyPath.toFile().exists());
-
-    // Read the private key and test if the expected String in the PEM file
-    // format exists.
-    byte[] privateKey = Files.readAllBytes(privateKeyPath);
-    String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
-    Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
-
-    // Read the public key and test if the expected String in the PEM file
-    // format exists.
-    byte[] publicKey = Files.readAllBytes(publicKeyPath);
-    String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
-    Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
-
-    // Let us decode the PEM file and parse it back into binary.
-    KeyFactory kf = KeyFactory.getInstance(
-        pemWriter.getSecurityConfig().getKeyAlgo());
-
-    // Replace the PEM Human readable guards.
-    privateKeydata =
-        privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
-    privateKeydata =
-        privateKeydata.replace("-----END PRIVATE KEY-----", "");
-
-    // Decode the bas64 to binary format and then use an ASN.1 parser to
-    // parse the binary format.
-
-    byte[] keyBytes = Base64.decodeBase64(privateKeydata);
-    PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
-    PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
-    Assert.assertNotNull("Private Key should not be null",
-        privateKeyDecoded);
-
-    // Let us decode the public key and veriy that we can parse it back into
-    // binary.
-    publicKeydata =
-        publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
-    publicKeydata =
-        publicKeydata.replace("-----END PUBLIC KEY-----", "");
-
-    keyBytes = Base64.decodeBase64(publicKeydata);
-    X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
-    PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
-    Assert.assertNotNull("Public Key should not be null",
-        publicKeyDecoded);
-
-    // Now let us assert the permissions on the Directories and files are as
-    // expected.
-    Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
-    Set<PosixFilePermission> currentSet =
-        Files.getPosixFilePermissions(privateKeyPath);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-
-    currentSet =
-        Files.getPosixFilePermissions(publicKeyPath);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-
-    currentSet =
-        Files.getPosixFilePermissions(keyLocation);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-  }
-
-  /**
-   * Assert key rewrite fails without force option.
-   *
-   * @throws IOException - on I/O failure.
-   */
-  @Test
-  public void testReWriteKey()
-      throws Exception {
-    KeyPair kp = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    SecurityConfig secConfig = pemWriter.getSecurityConfig();
-    pemWriter.writeKey(kp);
-
-    // Assert that rewriting of keys throws exception with valid messages.
-    LambdaTestUtils
-        .intercept(IOException.class, "Private Key file already exists.",
-            () -> pemWriter.writeKey(kp));
-    FileUtils.deleteQuietly(Paths.get(
-        secConfig.getKeyLocation().toString() + "/" + secConfig
-            .getPrivateKeyFileName()).toFile());
-    LambdaTestUtils
-        .intercept(IOException.class, "Public Key file already exists.",
-            () -> pemWriter.writeKey(kp));
-    FileUtils.deleteQuietly(Paths.get(
-        secConfig.getKeyLocation().toString() + "/" + secConfig
-            .getPublicKeyFileName()).toFile());
-
-    // Should succeed now as both public and private key are deleted.
-    pemWriter.writeKey(kp);
-    // Should succeed with overwrite flag as true.
-    pemWriter.writeKey(kp, true);
-
-  }
-
-  /**
-   * Assert key rewrite fails in non Posix file system.
-   *
-   * @throws IOException - on I/O failure.
-   */
-  @Test
-  public void testWriteKeyInNonPosixFS()
-      throws Exception {
-    KeyPair kp = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    pemWriter.setIsPosixFileSystem(() -> false);
-
-    // Assert key rewrite fails in non Posix file system.
-    LambdaTestUtils
-        .intercept(IOException.class, "Unsupported File System for pem file.",
-            () -> pemWriter.writeKey(kp));
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
new file mode 100644
index 0000000..f0973f7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestKeyCodec {
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private OzoneConfiguration configuration;
+  private HDDSKeyGenerator keyGenerator;
+  private String prefix;
+
+  @Before
+  public void init() throws IOException {
+    configuration = new OzoneConfiguration();
+    prefix = temporaryFolder.newFolder().toString();
+    configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+    keyGenerator = new HDDSKeyGenerator(configuration);
+  }
+
+  /**
+   * Assert basic things like we are able to create a file, and the names are
+   * in expected format etc.
+   *
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   * @throws IOException - On I/O failure.
+   */
+  @Test
+  public void testWriteKey()
+      throws NoSuchProviderException, NoSuchAlgorithmException,
+      IOException, InvalidKeySpecException {
+    KeyPair keys = keyGenerator.generateKey();
+    KeyCodec pemWriter = new KeyCodec(configuration);
+    pemWriter.writeKey(keys);
+
+    // Assert that locations have been created.
+    Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+    Assert.assertTrue(keyLocation.toFile().exists());
+
+    // Assert that locations are created in the locations that we specified
+    // using the Config.
+    Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+    Path privateKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPrivateKeyFileName());
+    Assert.assertTrue(privateKeyPath.toFile().exists());
+    Path publicKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPublicKeyFileName());
+    Assert.assertTrue(publicKeyPath.toFile().exists());
+
+    // Read the private key and test if the expected String in the PEM file
+    // format exists.
+    byte[] privateKey = Files.readAllBytes(privateKeyPath);
+    String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+    // Read the public key and test if the expected String in the PEM file
+    // format exists.
+    byte[] publicKey = Files.readAllBytes(publicKeyPath);
+    String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+    // Let us decode the PEM file and parse it back into binary.
+    KeyFactory kf = KeyFactory.getInstance(
+        pemWriter.getSecurityConfig().getKeyAlgo());
+
+    // Replace the PEM Human readable guards.
+    privateKeydata =
+        privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+    privateKeydata =
+        privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+    // Decode the bas64 to binary format and then use an ASN.1 parser to
+    // parse the binary format.
+
+    byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+    PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+    PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+    Assert.assertNotNull("Private Key should not be null",
+        privateKeyDecoded);
+
+    // Let us decode the public key and veriy that we can parse it back into
+    // binary.
+    publicKeydata =
+        publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+    publicKeydata =
+        publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+    keyBytes = Base64.decodeBase64(publicKeydata);
+    X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+    PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+    Assert.assertNotNull("Public Key should not be null",
+        publicKeyDecoded);
+
+    // Now let us assert the permissions on the Directories and files are as
+    // expected.
+    Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+    Set<PosixFilePermission> currentSet =
+        Files.getPosixFilePermissions(privateKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(publicKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(keyLocation);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+  }
+
+  /**
+   * Assert key rewrite fails without force option.
+   *
+   * @throws IOException - on I/O failure.
+   */
+  @Test
+  public void testReWriteKey()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    KeyCodec pemWriter = new KeyCodec(configuration);
+    SecurityConfig secConfig = pemWriter.getSecurityConfig();
+    pemWriter.writeKey(kp);
+
+    // Assert that rewriting of keys throws exception with valid messages.
+    LambdaTestUtils
+        .intercept(IOException.class, "Private Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPrivateKeyFileName()).toFile());
+    LambdaTestUtils
+        .intercept(IOException.class, "Public Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPublicKeyFileName()).toFile());
+
+    // Should succeed now as both public and private key are deleted.
+    pemWriter.writeKey(kp);
+    // Should succeed with overwrite flag as true.
+    pemWriter.writeKey(kp, true);
+
+  }
+
+  /**
+   * Assert key rewrite fails in non Posix file system.
+   *
+   * @throws IOException - on I/O failure.
+   */
+  @Test
+  public void testWriteKeyInNonPosixFS()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    KeyCodec pemWriter = new KeyCodec(configuration);
+    pemWriter.setIsPosixFileSystem(() -> false);
+
+    // Assert key rewrite fails in non Posix file system.
+    LambdaTestUtils
+        .intercept(IOException.class, "Unsupported File System for pem file.",
+            () -> pemWriter.writeKey(kp));
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
index 47ad597..d5695c5 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
@@ -31,7 +31,6 @@ import org.junit.rules.TemporaryFolder;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 
 /**
  * Tests RDBStore creation.
@@ -131,7 +130,7 @@ public class TestDBStoreBuilder {
             RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
         firstTable.put(key, value);
         byte[] temp = firstTable.get(key);
-        Assert.assertTrue(Arrays.equals(value, temp));
+        Assert.assertArrayEquals(value, temp);
       }
 
       try (Table secondTable = dbStore.getTable("Second")) {
@@ -161,7 +160,7 @@ public class TestDBStoreBuilder {
             RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
         firstTable.put(key, value);
         byte[] temp = firstTable.get(key);
-        Assert.assertTrue(Arrays.equals(value, temp));
+        Assert.assertArrayEquals(value, temp);
       }
 
       try (Table secondTable = dbStore.getTable("Second")) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index c1695f1..7fb995e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hdds.scm.server.SCMStorage;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
-import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyPEMWriter;
+import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.Client;
 import org.apache.hadoop.ipc.RPC;
@@ -86,7 +86,7 @@ public final class TestSecureOzoneCluster {
 
   private static final String TEST_USER = "testUgiUser";
   private static final int CLIENT_TIMEOUT = 2 * 1000;
-  private Logger LOGGER = LoggerFactory
+  private Logger logger = LoggerFactory
       .getLogger(TestSecureOzoneCluster.class);
 
   @Rule
@@ -125,9 +125,9 @@ public final class TestSecureOzoneCluster {
       createCredentialsInKDC(conf, miniKdc);
       generateKeyPair(conf);
     } catch (IOException e) {
-      LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+      logger.error("Failed to initialize TestSecureOzoneCluster", e);
     } catch (Exception e) {
-      LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+      logger.error("Failed to initialize TestSecureOzoneCluster", e);
     }
   }
 
@@ -146,7 +146,7 @@ public final class TestSecureOzoneCluster {
       }
       FileUtils.deleteQuietly(metaDirPath.toFile());
     } catch (Exception e) {
-      LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+      logger.error("Failed to stop TestSecureOzoneCluster", e);
     }
   }
 
@@ -449,7 +449,7 @@ public final class TestSecureOzoneCluster {
   private void generateKeyPair(OzoneConfiguration config) throws Exception {
     HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf);
     keyPair = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(config);
+    KeyCodec pemWriter = new KeyCodec(config);
     pemWriter.writeKey(keyPair, true);
   }
 
@@ -460,8 +460,6 @@ public final class TestSecureOzoneCluster {
    */
   @Test
   public void testDelegationTokenRenewal() throws Exception {
-    // Capture logs for assertions.
-    LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
     GenericTestUtils
         .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
 
@@ -477,7 +475,6 @@ public final class TestSecureOzoneCluster {
     om.start();
 
     UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    String username = ugi.getUserName();
 
     // Get first OM client which will authenticate via Kerberos
     omClient = new OzoneManagerProtocolClientSideTranslatorPB(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[08/50] [abbrv] hadoop git commit: YARN-8948. PlacementRule interface should be for all YarnSchedulers. Contributed by Bibin A Chundatt.

Posted by xy...@apache.org.
YARN-8948. PlacementRule interface should be for all YarnSchedulers. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a68d766e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a68d766e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a68d766e

Branch: refs/heads/HDDS-4
Commit: a68d766e876631d7ee2e1a6504d4120ba628d178
Parents: c1d24f8
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 21:43:34 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 21:43:34 2018 +0530

----------------------------------------------------------------------
 .../placement/AppNameMappingPlacementRule.java          | 12 ++++++++++--
 .../server/resourcemanager/placement/PlacementRule.java |  4 ++--
 .../placement/UserGroupMappingPlacementRule.java        | 11 ++++++++++-
 3 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
index 2debade..7a46962 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -61,8 +62,15 @@ public class AppNameMappingPlacementRule extends PlacementRule {
   }
 
   @Override
-  public boolean initialize(CapacitySchedulerContext schedulerContext)
+  public boolean initialize(ResourceScheduler scheduler)
       throws IOException {
+    if (!(scheduler instanceof CapacityScheduler)) {
+      throw new IOException(
+          "AppNameMappingPlacementRule can be configured only for "
+              + "CapacityScheduler");
+    }
+    CapacitySchedulerContext schedulerContext =
+        (CapacitySchedulerContext) scheduler;
     CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
     boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
     LOG.info(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
index 21ab32a..0f3d43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
@@ -22,7 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 
 public abstract class PlacementRule {
 
@@ -31,7 +31,7 @@ public abstract class PlacementRule {
   }
 
   public abstract boolean initialize(
-      CapacitySchedulerContext schedulerContext) throws IOException;
+      ResourceScheduler scheduler) throws IOException;
 
   /**
    * Get queue for a given application

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 99f37b1..a9e8235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -34,8 +34,10 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -230,8 +232,15 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
 
   @VisibleForTesting
   @Override
-  public boolean initialize(CapacitySchedulerContext schedulerContext)
+  public boolean initialize(ResourceScheduler scheduler)
       throws IOException {
+    if (!(scheduler instanceof CapacityScheduler)) {
+      throw new IOException(
+          "UserGroupMappingPlacementRule can be configured only for "
+              + "CapacityScheduler");
+    }
+    CapacitySchedulerContext schedulerContext =
+        (CapacitySchedulerContext) scheduler;
     CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
     boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
     LOG.info(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[31/50] [abbrv] hadoop git commit: HDDS-10. Add kdc docker image for secure ozone cluster. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-10. Add kdc docker image for secure ozone cluster. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/812c07ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/812c07ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/812c07ee

Branch: refs/heads/HDDS-4
Commit: 812c07ee4ffe110953d0d6b4ff3c1cb0287d569b
Parents: 585c344
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 4 13:20:09 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../dist/src/main/compose/ozonesecure/README.md | 22 +++++
 .../compose/ozonesecure/docker-compose.yaml     | 94 ++++++++++++--------
 .../docker-image/docker-krb5/Dockerfile-krb5    | 33 +++++++
 .../docker-image/docker-krb5/README.md          | 34 +++++++
 .../docker-image/docker-krb5/kadm5.acl          |  1 +
 .../docker-image/docker-krb5/krb5.conf          | 40 +++++++++
 .../docker-image/docker-krb5/launcher.sh        | 25 ++++++
 7 files changed, 210 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
new file mode 100644
index 0000000..0ce9a0a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
@@ -0,0 +1,22 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+#### Dockerfile for KDC:
+* ./docker-image/docker-krb5/Dockerfile-krb5
+
+#### Dockerfile for SCM,OM and DataNode:
+* ./docker-image/runner/Dockerfile
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 42ab05e..fab5ba9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -16,42 +16,58 @@
 
 version: "3"
 services:
-   kdc:
-      image: ahadoop/kdc:v1
-      hostname: kdc
-      volumes:
-          - $SRC_VOLUME:/opt/hadoop
-   datanode:
-      image: ahadoop/runner:latest
-      volumes:
-        - $SRC_VOLUME:/opt/hadoop
-      hostname: datanode
-      ports:
-        - 9864
-      command: ["/opt/hadoop/bin/ozone","datanode"]
-      env_file:
-        - ./docker-config
-   ozoneManager:
-      image: ahadoop/runner:latest
-      hostname: om
-      volumes:
-         - $SRC_VOLUME:/opt/hadoop
-      ports:
-         - 9874:9874
-      environment:
-         ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
-      env_file:
-          - ./docker-config
-      command: ["/opt/hadoop/bin/ozone","om"]
-   scm:
-      image: ahadoop/runner:latest
-      hostname: scm
-      volumes:
-         - $SRC_VOLUME:/opt/hadoop
-      ports:
-         - 9876:9876
-      env_file:
-          - ./docker-config
-      environment:
-          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["/opt/hadoop/bin/ozone","scm"]
+  kdc:
+    build:
+      context: docker-image/docker-krb5
+      dockerfile: Dockerfile-krb5
+      args:
+        buildno: 1
+    hostname: kdc
+    volumes:
+    - $SRC_VOLUME:/opt/hadoop
+  datanode:
+    build:
+      context: docker-image/runner
+      dockerfile: Dockerfile
+      args:
+        buildno: 1
+    volumes:
+    - $SRC_VOLUME:/opt/hadoop
+    hostname: datanode
+    ports:
+    - 9864
+    command: ["/opt/hadoop/bin/ozone","datanode"]
+    env_file:
+    - docker-config
+  om:
+    build:
+      context: docker-image/runner
+      dockerfile: Dockerfile
+      args:
+        buildno: 1
+    hostname: om
+    volumes:
+    - $SRC_VOLUME:/opt/hadoop
+    ports:
+    - 9874:9874
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+    env_file:
+    - docker-config
+    command: ["/opt/hadoop/bin/ozone","om"]
+  scm:
+    build:
+      context: docker-image/runner
+      dockerfile: Dockerfile
+      args:
+        buildno: 1
+    hostname: scm
+    volumes:
+    - $SRC_VOLUME:/opt/hadoop
+    ports:
+    - 9876:9876
+    env_file:
+    - docker-config
+    environment:
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+    command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
new file mode 100644
index 0000000..b5b931d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+
+FROM frolvlad/alpine-oraclejdk8:slim
+RUN apk add --update bash ca-certificates openssl krb5-server krb5 && rm -rf /var/cache/apk/* && update-ca-certificates
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN wget -O /root/issuer https://github.com/ajayydv/docker/raw/kdc/issuer
+RUN chmod +x /root/issuer
+WORKDIR /opt
+ADD krb5.conf /etc/
+ADD kadm5.acl /var/lib/krb5kdc/kadm5.acl
+RUN kdb5_util create -s -P Welcome1
+RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
+RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
+ADD launcher.sh .
+RUN mkdir -p /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
new file mode 100644
index 0000000..b864a5f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
@@ -0,0 +1,34 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+The docker image contains a rest service which provides keystore and keytab files without any authentication!
+
+Master password: Welcome1
+
+Principal: admin/admin@EXAMPLE.COM Password: Welcome1
+
+Test:
+
+```
+docker run --net=host krb5
+
+docker run --net=host -it --entrypoint=bash krb5
+kinit admin/admin 
+#pwd: Welcome1
+klist
+```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
new file mode 100644
index 0000000..8fe9f69
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -0,0 +1 @@
+*/admin@EXAMPLE.COM x

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
new file mode 100644
index 0000000..34eabe1
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+default = FILE:/var/log/krb5libs.log
+kdc = FILE:/var/log/krb5kdc.log
+admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+  kdc = localhost
+  admin_server = localhost
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
new file mode 100644
index 0000000..0824f7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+/root/issuer &
+krb5kdc -n &
+sleep 4
+kadmind -nofork &
+sleep 2
+tail -f /var/log/krb5kdc.log &
+tail -f /var/log/kadmind.log
+


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[42/50] [abbrv] hadoop git commit: HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bbc95ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bbc95ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bbc95ee

Branch: refs/heads/HDDS-4
Commit: 8bbc95ee536caae21f28f9513fe624955aa122a0
Parents: 9128c66
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 6 16:53:04 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../acceptance/ozone-secure.robot               |  95 ----------------
 .../dist/src/main/compose/ozonesecure/.env      |   1 -
 .../compose/ozonesecure/docker-compose.yaml     |  22 ++--
 .../ozonesecure/docker-image/runner/Dockerfile  |   4 +-
 .../main/smoketest/security/ozone-secure.robot  | 111 +++++++++++++++++++
 hadoop-ozone/dist/src/main/smoketest/test.sh    |   2 +
 6 files changed, 126 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
deleted file mode 100644
index 7fc1088..0000000
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation       Smoke test to start cluster with docker-compose environments.
-Library             OperatingSystem
-Suite Setup         Startup Ozone Cluster
-Suite Teardown      Teardown Ozone Cluster
-
-*** Variables ***
-${COMMON_REST_HEADER}   -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H  "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${version}
-
-*** Test Cases ***
-
-Daemons are running
-    Is daemon running           om
-    Is daemon running           scm
-    Is daemon running           datanode
-    Is daemon running           ozone.kdc
-
-Check if datanode is connected to the scm
-    Wait Until Keyword Succeeds     3min    5sec    Have healthy datanodes   1
-
-Test rest interface
-    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
-                    Should contain      ${result}       201 Created
-    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
-                    Should contain      ${result}       201 Created
-    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
-                    Should contain      ${result}       200 OK
-    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
-                    Should contain      ${result}       200 OK
-
-Test ozone cli
-    ${result} =     Execute on      1   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
-                    Should contain      ${result}       Client cannot authenticate via
-                    # Authenticate testuser
-                    Execute on      0   datanode        kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
-                    Execute on      0   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
-    ${result} =     Execute on      0   datanode        ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
-                    Should contain      ${result}       createdOn
-                    Execute on      0   datanode        ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
-    ${result} =     Execute on      0   datanode        ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
-                    Should Be Equal     ${result}       bill
-
-*** Keywords ***
-
-Startup Ozone Cluster
-    ${rc}       ${output} =                 Run docker compose        0     down
-    ${rc}       ${output} =                 Run docker compose        0     up -d
-    Should Be Equal As Integers 	          ${rc} 	                  0
-    Wait Until Keyword Succeeds             3min    10sec    Is Daemon started   ksm     KSM is listening
-
-Teardown Ozone Cluster
-    Run docker compose      0           down
-
-Is daemon running
-    [arguments]             ${name}
-    ${result} =             Run                     docker ps
-    Should contain          ${result}               _${name}_1
-
-Is Daemon started
-    [arguments]     ${name}             ${expression}
-    ${rc}           ${result} =         Run docker compose      0         logs
-    Should contain  ${result}           ${expression}
-
-Have healthy datanodes
-    [arguments]         ${requirednodes}
-    ${result} =         Execute on     0     scm                 curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
-    Should Be Equal     ${result}           ${requirednodes}
-
-Execute on
-    [arguments]     ${expected_rc}      ${componentname}      ${command}
-    ${rc}           ${return} =         Run docker compose    ${expected_rc}     exec ${componentname} ${command}
-    [return]        ${return}
-
-Run docker compose
-    [arguments]                     ${expected_rc}              ${command}
-                                    Set Environment Variable    OZONEDIR                               ${basedir}/hadoop-dist/target/ozone
-    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
-    Should Be Equal As Integers     ${rc}                       ${expected_rc}
-    [return]                        ${rc}                       ${output}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index a494004..cac418a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,4 +15,3 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index fab5ba9..034d0ff 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
         buildno: 1
     hostname: kdc
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
   datanode:
     build:
       context: docker-image/runner
@@ -32,13 +32,13 @@ services:
       args:
         buildno: 1
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     hostname: datanode
     ports:
-    - 9864
+      - 9864
     command: ["/opt/hadoop/bin/ozone","datanode"]
     env_file:
-    - docker-config
+      - docker-config
   om:
     build:
       context: docker-image/runner
@@ -47,13 +47,13 @@ services:
         buildno: 1
     hostname: om
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     ports:
-    - 9874:9874
+      - 9874:9874
     environment:
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
     env_file:
-    - docker-config
+      - docker-config
     command: ["/opt/hadoop/bin/ozone","om"]
   scm:
     build:
@@ -63,11 +63,11 @@ services:
         buildno: 1
     hostname: scm
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     ports:
-    - 9876:9876
+      - 9876:9876
     env_file:
-    - docker-config
+      - docker-config
     environment:
       ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-    command: ["/opt/hadoop/bin/ozone","scm"]
+    command: ["/opt/hadoop/bin/ozone","scm"]
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
index efda03e..17eff18 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -15,8 +15,8 @@
 # limitations under the License.
 
 FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y  jq curl python sudo && apt-get clean
-
+RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean
+RUN pip install robotframework
 RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
 RUN chmod +x /usr/local/bin/dumb-init
 RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
new file mode 100644
index 0000000..457ebaa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test to start cluster with docker-compose environments.
+Library             OperatingSystem
+Resource            ../commonlib.robot
+
+*** Test Cases ***
+Create volume and bucket
+    ${rc}              ${output} =                 Run And Return Rc And Output              ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+                        Should contain       ${output}       Client cannot authenticate via
+                        # Authenticate testuser
+    Execute             kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+    Execute             ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+    Execute             ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root
+    Execute             ozone sh bucket create o3://om/fstest/bucket1
+    Execute             ozone sh bucket create o3://om/fstest/bucket2
+    Execute             ozone sh bucket create o3://om/fstest2/bucket3
+
+Check volume from ozonefs
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/
+
+Run ozoneFS tests
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         testdir/deep
+                        Execute               ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         NOTICE.txt
+
+                        Execute               ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         PUTFILE.txt
+
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+                        Should contain    ${result}         NOTICE.txt
+                        Should contain    ${result}         PUTFILE.txt
+
+                        Execute               ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         MOVED.TXT
+                        Should not contain  ${result}       NOTICE.txt
+
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         subdir1/NOTICE.txt
+
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
+                        Should contain    ${result}         NOTICE.txt
+
+                        Execute               ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+                        Should not contain  ${result}       Failed
+
+                        Execute               ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       NOTICE.txt
+
+    ${result} =         Execute               ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       subdir1
+
+                        Execute               ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain  ${result}       TOUCHFILE.txt
+
+                        Execute               ozone fs -rm -r o3fs://bucket1.fstest/testdir/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       testdir
+
+                        Execute               rm -Rf localdir1
+                        Execute               mkdir localdir1
+                        Execute               cp NOTICE.txt localdir1/LOCAL.txt
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
+                        Execute               ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
+                        Execute               ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
+
+    ${result} =         Execute               ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
+                        Should contain    ${result}         localdir1/LOCAL.txt
+                        Should contain    ${result}         testdir1/NOTICE.txt
+
+                        Execute               ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
+                        Execute               ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
+                        Execute               ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/KEY.txt
+                        Should contain    ${result}         KEY.txt
+    ${rc}  ${result} =  Run And Return Rc And Output        ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
+                        Should Be Equal As Integers     ${rc}                1
+                        Should contain    ${result}         File exists
+                        Execute               rm -Rf GET.txt
+                        Execute               ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
+                        Execute               ls -l GET.txt
+    ${rc}  ${result} =  Run And Return Rc And Output        ozone fs -ls o3fs://abcde.pqrs/
+                        Should Be Equal As Integers     ${rc}                1
+                        Should contain    ${result}         VOLUME_NOT_FOUND
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
index 5819595..53d0876 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then
 #
 # We select the test suites and execute them on multiple type of clusters
 #
+   DEFAULT_TESTS=("security")
+   execute_tests ozonesecure "${DEFAULT_TESTS[@]}"
    DEFAULT_TESTS=("basic")
    execute_tests ozone "${DEFAULT_TESTS[@]}"
    TESTS=("ozonefs")


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[15/50] [abbrv] hadoop git commit: HDDS-877. Ensure correct surefire version for Ozone test. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-877. Ensure correct surefire version for Ozone test. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae5fbdd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae5fbdd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae5fbdd9

Branch: refs/heads/HDDS-4
Commit: ae5fbdd9ed6ef09b588637f2eadd7a04e8382289
Parents: f534736
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Nov 29 11:37:36 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:37:36 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/pom.xml  | 1 +
 hadoop-ozone/pom.xml | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 869ecbf..5537b3a 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -53,6 +53,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
 
+    <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
 
   </properties>
   <repositories>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 39c65d5..4c13bd6 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -37,8 +37,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <bouncycastle.version>1.60</bouncycastle.version>
     <ozone.release>Badlands</ozone.release>
     <declared.ozone.version>${ozone.version}</declared.ozone.version>
-
-
+    <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
   </properties>
   <modules>
     <module>common</module>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[13/50] [abbrv] hadoop git commit: HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"

Posted by xy...@apache.org.
HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"

This reverts commit 829a2e4d271f05afb209ddc834cd4a0e85492eda.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0edd372
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0edd372
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0edd372

Branch: refs/heads/HDDS-4
Commit: d0edd37269bb40290b409d583bcf3b70897c13e0
Parents: 5e102f9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 29 17:52:11 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 17:52:11 2018 +0000

----------------------------------------------------------------------
 .../authentication/util/KerberosName.java       |  9 ++--
 .../TestKerberosAuthenticationHandler.java      |  7 ++-
 .../authentication/util/TestKerberosName.java   | 17 ++++++--
 .../java/org/apache/hadoop/security/KDiag.java  | 46 +-------------------
 .../src/site/markdown/SecureMode.md             |  6 ---
 .../org/apache/hadoop/security/TestKDiag.java   | 16 -------
 .../security/TestUserGroupInformation.java      | 27 ++++--------
 7 files changed, 33 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
index 4e7ee3c..287bb13 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
@@ -324,8 +324,8 @@ public class KerberosName {
         }
       }
       if (result != null && nonSimplePattern.matcher(result).find()) {
-        LOG.info("Non-simple name {} after auth_to_local rule {}",
-            result, this);
+        throw new NoMatchingRule("Non-simple name " + result +
+                                 " after auth_to_local rule " + this);
       }
       if (toLowerCase && result != null) {
         result = result.toLowerCase(Locale.ENGLISH);
@@ -378,7 +378,7 @@ public class KerberosName {
   /**
    * Get the translation of the principal name into an operating system
    * user name.
-   * @return the user name
+   * @return the short name
    * @throws IOException throws if something is wrong with the rules
    */
   public String getShortName() throws IOException {
@@ -398,8 +398,7 @@ public class KerberosName {
         return result;
       }
     }
-    LOG.info("No auth_to_local rules applied to {}", this);
-    return toString();
+    throw new NoMatchingRule("No rules applied to " + toString());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
index e672391..8b4bc15 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
@@ -108,7 +108,12 @@ public class TestKerberosAuthenticationHandler
     kn = new KerberosName("bar@BAR");
     Assert.assertEquals("bar", kn.getShortName());
     kn = new KerberosName("bar@FOO");
-    Assert.assertEquals("bar@FOO", kn.getShortName());
+    try {
+      kn.getShortName();
+      Assert.fail();
+    }
+    catch (Exception ex) {      
+    }
   }
 
   @Test(timeout=60000)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
index c584fce..2db0df4 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
@@ -72,14 +72,23 @@ public class TestKerberosName {
     }
   }
 
+  private void checkBadTranslation(String from) {
+    System.out.println("Checking bad translation for " + from);
+    KerberosName nm = new KerberosName(from);
+    try {
+      nm.getShortName();
+      Assert.fail("didn't get exception for " + from);
+    } catch (IOException ie) {
+      // PASS
+    }
+  }
+
   @Test
   public void testAntiPatterns() throws Exception {
     checkBadName("owen/owen/owen@FOO.COM");
     checkBadName("owen@foo/bar.com");
-
-    // no rules applied, these should pass
-    checkTranslation("foo@ACME.COM", "foo@ACME.COM");
-    checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM");
+    checkBadTranslation("foo@ACME.COM");
+    checkBadTranslation("root/joe@FOO.COM");
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index b4e535c..1711921 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -22,7 +22,6 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.ExitUtil;
@@ -55,7 +54,6 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
 import static org.apache.hadoop.security.UserGroupInformation.*;
@@ -131,12 +129,6 @@ public class KDiag extends Configured implements Tool, Closeable {
   private boolean nofail = false;
   private boolean nologin = false;
   private boolean jaas = false;
-  private boolean checkShortName = false;
-
-  /**
-   * A pattern that recognizes simple/non-simple names. Per KerberosName
-   */
-  private static final Pattern nonSimplePattern = Pattern.compile("[/@]");
 
   /**
    * Flag set to true if a {@link #verify(boolean, String, String, Object...)}
@@ -165,8 +157,6 @@ public class KDiag extends Configured implements Tool, Closeable {
 
   public static final String ARG_SECURE = "--secure";
 
-  public static final String ARG_VERIFYSHORTNAME = "--verifyshortname";
-
   @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
   public KDiag(Configuration conf,
       PrintWriter out,
@@ -210,7 +200,6 @@ public class KDiag extends Configured implements Tool, Closeable {
     nofail = popOption(ARG_NOFAIL, args);
     jaas = popOption(ARG_JAAS, args);
     nologin = popOption(ARG_NOLOGIN, args);
-    checkShortName = popOption(ARG_VERIFYSHORTNAME, args);
 
     // look for list of resources
     String resource;
@@ -256,9 +245,7 @@ public class KDiag extends Configured implements Tool, Closeable {
       + arg(ARG_NOLOGIN, "", "Do not attempt to log in")
       + arg(ARG_OUTPUT, "<file>", "Write output to a file")
       + arg(ARG_RESOURCE, "<resource>", "Load an XML configuration resource")
-      + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure")
-      + arg(ARG_VERIFYSHORTNAME, ARG_PRINCIPAL + " <principal>",
-      "Verify the short name of the specific principal does not contain '@' or '/'");
+      + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure");
   }
 
   private String arg(String name, String params, String meaning) {
@@ -291,7 +278,6 @@ public class KDiag extends Configured implements Tool, Closeable {
     println("%s = %d", ARG_KEYLEN, minKeyLength);
     println("%s = %s", ARG_KEYTAB, keytab);
     println("%s = %s", ARG_PRINCIPAL, principal);
-    println("%s = %s", ARG_VERIFYSHORTNAME, checkShortName);
 
     // Fail fast on a JVM without JCE installed.
     validateKeyLength();
@@ -391,10 +377,6 @@ public class KDiag extends Configured implements Tool, Closeable {
       validateJAAS(jaas);
       validateNTPConf();
 
-      if (checkShortName) {
-        validateShortName();
-      }
-
       if (!nologin) {
         title("Logging in");
         if (keytab != null) {
@@ -449,32 +431,6 @@ public class KDiag extends Configured implements Tool, Closeable {
   }
 
   /**
-   * Verify whether auth_to_local rules transform a principal name
-   * <p>
-   * Having a local user name "bar@foo.com" may be harmless, so it is noted at
-   * info. However if what was intended is a transformation to "bar"
-   * it can be difficult to debug, hence this check.
-   */
-  protected void validateShortName() {
-    failif(principal == null, CAT_KERBEROS, "No principal defined");
-
-    try {
-      KerberosName kn = new KerberosName(principal);
-      String result = kn.getShortName();
-      if (nonSimplePattern.matcher(result).find()) {
-        warn(CAT_KERBEROS, principal + " short name: " + result
-                + " still contains @ or /");
-      }
-    } catch (IOException e) {
-      throw new KerberosDiagsFailure(CAT_KERBEROS, e,
-              "Failed to get short name for " + principal, e);
-    } catch (IllegalArgumentException e) {
-      error(CAT_KERBEROS, "KerberosName(" + principal + ") failed: %s\n%s",
-              e, StringUtils.stringifyException(e));
-    }
-  }
-
-  /**
    * Get the default realm.
    * <p>
    * Not having a default realm may be harmless, so is noted at info.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index ecf94eb..09f1eed 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -470,7 +470,6 @@ KDiag: Diagnose Kerberos Problems
   [--out <file>] : Write output to a file.
   [--resource <resource>] : Load an XML configuration resource.
   [--secure] : Require the hadoop configuration to be secure.
-  [--verifyshortname <principal>]: Verify the short name of the specific principal does not contain '@' or '/'
 ```
 
 #### `--jaas`: Require a JAAS file to be defined in `java.security.auth.login.config`.
@@ -566,11 +565,6 @@ or implicitly set to "simple":
 
 Needless to say, an application so configured cannot talk to a secure Hadoop cluster.
 
-#### `--verifyshortname <principal>`: validate the short name of a principal
-
-This verifies that the short name of a principal contains neither the `"@"`
-nor `"/"` characters.
-
 ### Example
 
 ```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
index e395566..3895ae1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
@@ -165,22 +165,6 @@ public class TestKDiag extends Assert {
   }
 
   @Test
-  public void testKerberosName() throws Throwable {
-    kdiagFailure(ARG_KEYLEN, KEYLEN,
-            ARG_VERIFYSHORTNAME,
-            ARG_PRINCIPAL, "foo/foo/foo@BAR.COM");
-  }
-
-  @Test
-  public void testShortName() throws Throwable {
-    kdiag(ARG_KEYLEN, KEYLEN,
-            ARG_KEYTAB, keytab.getAbsolutePath(),
-            ARG_PRINCIPAL,
-            ARG_VERIFYSHORTNAME,
-            ARG_PRINCIPAL, "foo@EXAMPLE.COM");
-  }
-
-  @Test
   public void testFileOutput() throws Throwable {
     File f = new File("target/kdiag.txt");
     kdiag(ARG_KEYLEN, KEYLEN,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 3020f9b..8af0328 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -332,15 +332,10 @@ public class TestUserGroupInformation {
     UserGroupInformation.setConfiguration(conf);
     testConstructorSuccess("user1", "user1");
     testConstructorSuccess("user4@OTHER.REALM", "other-user4");
-
-    // pass through test, no transformation
-    testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM");
-    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM");
-    testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
-    // failures
-    testConstructorFailures("user6@example.com@OTHER.REALM");
-    testConstructorFailures("user7@example.com@DEFAULT.REALM");
+    // failure test
+    testConstructorFailures("user2@DEFAULT.REALM");
+    testConstructorFailures("user3/cron@DEFAULT.REALM");
+    testConstructorFailures("user5/cron@OTHER.REALM");
     testConstructorFailures(null);
     testConstructorFailures("");
   }
@@ -354,13 +349,10 @@ public class TestUserGroupInformation {
 
     testConstructorSuccess("user1", "user1");
     testConstructorSuccess("user2@DEFAULT.REALM", "user2");
-    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
-
-    // no rules applied, local name remains the same
-    testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM");
-    testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
+    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");    
     // failure test
+    testConstructorFailures("user4@OTHER.REALM");
+    testConstructorFailures("user5/cron@OTHER.REALM");
     testConstructorFailures(null);
     testConstructorFailures("");
   }
@@ -401,9 +393,8 @@ public class TestUserGroupInformation {
     } catch (IllegalArgumentException e) {
       String expect = (userName == null || userName.isEmpty())
           ? "Null user" : "Illegal principal name "+userName;
-      String expect2 = "Malformed Kerberos name: "+userName;
-      assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e,
-          e.toString().contains(expect) || e.toString().contains(expect2));
+      assertTrue("Did not find "+ expect + " in " + e,
+          e.toString().contains(expect));
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[30/50] [abbrv] hadoop git commit: HDDS-547. Fix secure docker and configs. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-547. Fix secure docker and configs. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eddbe997
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eddbe997
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eddbe997

Branch: refs/heads/HDDS-4
Commit: eddbe997d86c54c21763b45de8d7f4a89a97dfa2
Parents: 87acc15
Author: Ajay Kumar <aj...@apache.com>
Authored: Mon Oct 1 11:03:27 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 hadoop-dist/src/main/compose/ozonesecure/.env   |  18 +++
 .../compose/ozonesecure/docker-compose.yaml     |  57 +++++++
 .../src/main/compose/ozonesecure/docker-config  | 103 +++++++++++++
 .../ozonesecure/docker-image/runner/Dockerfile  |  39 +++++
 .../ozonesecure/docker-image/runner/build.sh    |  26 ++++
 .../docker-image/runner/scripts/envtoconf.py    | 115 ++++++++++++++
 .../docker-image/runner/scripts/krb5.conf       |  38 +++++
 .../docker-image/runner/scripts/starter.sh      | 100 +++++++++++++
 .../runner/scripts/transformation.py            | 150 +++++++++++++++++++
 hadoop-hdds/common/pom.xml                      |   6 +
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |   8 +-
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |   3 -
 .../common/src/main/resources/ozone-default.xml |   6 +-
 .../hadoop/ozone/HddsDatanodeService.java       |  30 ++++
 .../StorageContainerManagerHttpServer.java      |   4 +-
 .../src/test/compose/compose-secure/.env        |  17 ---
 .../compose/compose-secure/docker-compose.yaml  |  66 --------
 .../test/compose/compose-secure/docker-config   |  99 ------------
 .../apache/hadoop/ozone/om/OMConfigKeys.java    |   4 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    |  11 +-
 .../hadoop/ozone/om/OzoneManagerHttpServer.java |   4 +-
 21 files changed, 701 insertions(+), 203 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   kdc:
+      image: ahadoop/kdc:v1
+      hostname: kdc
+      volumes:
+          - $SRC_VOLUME:/opt/hadoop
+   datanode:
+      image: ahadoop/runner:latest
+      volumes:
+        - $SRC_VOLUME:/opt/hadoop
+      hostname: datanode
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+      env_file:
+        - ./docker-config
+   ozoneManager:
+      image: ahadoop/runner:latest
+      hostname: om
+      volumes:
+         - $SRC_VOLUME:/opt/hadoop
+      ports:
+         - 9874:9874
+      environment:
+         ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","om"]
+   scm:
+      image: ahadoop/runner:latest
+      hostname: scm
+      volumes:
+         - $SRC_VOLUME:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y  jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+  wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+  cd "$DIR"/build || exit
+  tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+  """Simple conversion"""
+  def __init__(self, args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--destination", help="Destination directory", required=True)
+    self.args = parser.parse_args(args=args)
+    # copy the default files to file.raw in destination directory
+
+    self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+    self.output_dir = self.args.destination
+
+    self.configurables = {}
+
+  def destination_file_path(self, name, extension):
+    """destination file path"""
+    return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+  def write_env_var(self, name, extension, key, value):
+    """Write environment variables"""
+    with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+      myfile.write("{}: {}\n".format(key, value))
+
+  def process_envs(self):
+    """Process environment variables"""
+    for key in os.environ.keys():
+      pattern = re.compile("[_\\.]")
+      parts = pattern.split(key)
+      extension = None
+      name = parts[0].lower()
+      if len(parts) > 1:
+        extension = parts[1].lower()
+        config_key = key[len(name) + len(extension) + 2:].strip()
+      if extension and "!" in extension:
+        splitted = extension.split("!")
+        extension = splitted[0]
+        fmt = splitted[1]
+        config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+      else:
+        fmt = extension
+
+      if extension and extension in self.known_formats:
+        if name not in self.configurables.keys():
+          with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+            myfile.write("")
+        self.configurables[name] = (extension, fmt)
+        self.write_env_var(name, extension, config_key, os.environ[key])
+      else:
+        for configurable_name in self.configurables:
+          if key.lower().startswith(configurable_name.lower()):
+            self.write_env_var(configurable_name,
+                               self.configurables[configurable_name],
+                               key[len(configurable_name) + 1:],
+                               os.environ[key])
+
+  def transform(self):
+    """transform"""
+    for configurable_name in self.configurables:
+      name = configurable_name
+      extension, fmt = self.configurables[name]
+
+      destination_path = self.destination_file_path(name, extension)
+
+      with open(destination_path + ".raw", "r") as myfile:
+        content = myfile.read()
+        transformer_func = getattr(transformation, "to_" + fmt)
+        content = transformer_func(content)
+        with open(destination_path, "w") as myfile:
+          myfile.write(content)
+
+  def main(self):
+    """main"""
+
+    # add the
+    self.process_envs()
+
+    # copy file.ext.raw to file.ext in the destination directory, and
+    # transform to the right format (eg. key: value ===> XML)
+    self.transform()
+
+
+def main():
+  """main"""
+  Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+  Simple(sys.argv[1:]).main()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+  kdc = SERVER
+  admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+  echo "Setting up kerberos!!"
+  KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+  ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+  echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+  if [ -n "$SLEEP_SECONDS" ]; then
+    echo "Sleeping for $(SLEEP_SECONDS) seconds"
+    sleep "$SLEEP_SECONDS"
+  fi
+
+
+  while true
+    do
+      STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+      if [ "$STATUS" -eq 200 ]; then
+        echo "Got 200, KDC service ready!!"
+        break
+      else
+        echo "Got $STATUS :( KDC service not ready yet..."
+      fi
+      sleep 5
+    done
+
+    HOST_NAME=$(hostname -f)
+    export HOST_NAME
+    for NAME in ${KERBEROS_KEYTABS}; do
+      echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+      wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+      klist -kt "$CONF_DIR/$NAME.keytab"
+      KERBEROS_ENABLED=true
+    done
+
+    sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+  CLUSTERID_OPTS=""
+  if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+    CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+  fi
+  if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+    /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+  fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+  if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+    /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+  fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+  if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+    /opt/hadoop/bin/ozone scm -init
+  fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+  if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+    #To make sure SCM is running in dockerized environment we will sleep
+    # Could be removed after HDFS-13203
+    echo "Waiting 15 seconds for SCM startup"
+    sleep 15
+    /opt/hadoop/bin/ozone om -createObjectStore
+  fi
+fi
+
+echo 'setup finished'
+"$@"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+  """render yaml"""
+  result = ""
+  if isinstance(yaml_root, dict):
+    if prefix:
+      result += "\n"
+      for key in yaml_root:
+        result += "{}{}: {}".format(prefix, key, render_yaml(
+            yaml_root[key], prefix + "   "))
+  elif isinstance(yaml_root, list):
+    result += "\n"
+    for item in yaml_root:
+      result += prefix + " - " + render_yaml(item, prefix + " ")
+  else:
+    result += "{}\n".format(yaml_root)
+  return result
+
+
+def to_yaml(content):
+  """transform to yaml"""
+  props = process_properties(content)
+
+  keys = props.keys()
+  yaml_props = {}
+  for key in keys:
+    parts = key.split(".")
+    node = yaml_props
+    prev_part = None
+    parent_node = {}
+    for part in parts[:-1]:
+      if part.isdigit():
+        if isinstance(node, dict):
+          parent_node[prev_part] = []
+          node = parent_node[prev_part]
+        while len(node) <= int(part):
+          node.append({})
+        parent_node = node
+        node = node[int(node)]
+      else:
+        if part not in node:
+          node[part] = {}
+        parent_node = node
+        node = node[part]
+      prev_part = part
+    if parts[-1].isdigit():
+      if isinstance(node, dict):
+        parent_node[prev_part] = []
+        node = parent_node[prev_part]
+      node.append(props[key])
+    else:
+      node[parts[-1]] = props[key]
+
+  return render_yaml(yaml_props)
+
+
+def to_yml(content):
+  """transform to yml"""
+  return to_yaml(content)
+
+
+def to_properties(content):
+  """transform to properties"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props.items():
+    result += "{}: {}\n".format(key, val)
+  return result
+
+
+def to_env(content):
+  """transform to environment variables"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "{}={}\n".format(key, val)
+  return result
+
+
+def to_sh(content):
+  """transform to shell"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "export {}=\"{}\"\n".format(key, val)
+  return result
+
+
+def to_cfg(content):
+  """transform to config"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "{}={}\n".format(key, val)
+  return result
+
+
+def to_conf(content):
+  """transform to configuration"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "export {}={}\n".format(key, val)
+  return result
+
+
+def to_xml(content):
+  """transform to xml"""
+  result = "<configuration>\n"
+  props = process_properties(content)
+  for key in props:
+    result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+      format(key, props[key])
+  result += "</configuration>"
+  return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+  """
+  Read the file passed as parameter as a properties file.
+  """
+  props = {}
+  for line in content.split("\n"):
+    sline = line.strip()
+    if sline and not sline.startswith(comment_char):
+      key_value = sline.split(sep)
+      key = key_value[0].strip()
+      value = sep.join(key_value[1:]).strip().strip('"')
+      props[key] = value
+
+  return props

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 15ae307..30dcc58 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -118,6 +118,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>bcprov-jdk15on</artifactId>
       <version>1.54</version>
     </dependency>
+
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk15on</artifactId>
+      <version>1.54</version>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index cb6672f..abaa0f8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -314,11 +314,11 @@ public final class ScmConfigKeys {
       "10m";
 
   public static final String
-      HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
-      "hdds.scm.web.authentication.kerberos.principal";
+      HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
+      "hdds.scm.http.kerberos.principal";
   public static final String
-      HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
-      "hdds.scm.web.authentication.kerberos.keytab";
+      HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
+      "hdds.scm.http.kerberos.keytab";
   /**
    * Never constructed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index b47113b..ac97422 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -311,9 +311,6 @@ public final class OzoneConfigKeys {
       DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT =
       ScmConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT;
 
-  public static final String OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
-      "ozone.web.authentication.kerberos.principal";
-
   public static final String HDDS_DATANODE_PLUGINS_KEY =
       "hdds.datanode.plugins";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7d91dc8..5fb594b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1430,11 +1430,11 @@
   </property>
 
   <property>
-    <name>hdds.scm.web.authentication.kerberos.principal</name>
+    <name>hdds.scm.http.kerberos.principal</name>
     <value>HTTP/_HOST@EXAMPLE.COM</value>
   </property>
   <property>
-    <name>hdds.scm.web.authentication.kerberos.keytab</name>
+    <name>hdds.scm.http.kerberos.keytab</name>
     <value>/etc/security/keytabs/HTTP.keytab</value>
   </property>
 
@@ -1446,7 +1446,7 @@
     </description>
   </property>
   <property>
-    <name>ozone.om.http.kerberos.keytab.file</name>
+    <name>ozone.om.http.kerberos.keytab</name>
     <value>/etc/security/keytabs/HTTP.keytab</value>
     <description>
       OzoneManager http server kerberos keytab.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 348196c..8865368 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,11 +25,15 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.statemachine
     .DatanodeStateMachine;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ServicePlugin;
 import org.apache.hadoop.util.StringUtils;
@@ -98,12 +102,38 @@ public class HddsDatanodeService implements ServicePlugin {
         datanodeDetails = initializeDatanodeDetails();
         datanodeDetails.setHostName(hostname);
         datanodeDetails.setIpAddress(ip);
+        LOG.info("HddsDatanodeService host:{} ip:{}", hostname, ip);
+        // Authenticate Hdds Datanode service if security is enabled
+        if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY,
+            true)) {
+          if (SecurityUtil.getAuthenticationMethod(conf).equals(
+              UserGroupInformation.AuthenticationMethod.KERBEROS)) {
+            LOG.debug("Ozone security is enabled. Attempting login for Hdds " +
+                    "Datanode user. "
+                    + "Principal: {},keytab: {}", conf.get(
+                DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY),
+                conf.get(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY));
+
+            UserGroupInformation.setConfiguration(conf);
+
+            SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY,
+                DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hostname);
+          } else {
+            throw new AuthenticationException(SecurityUtil.
+                getAuthenticationMethod(conf) + " authentication method not " +
+                "supported. Datanode user" + " login " + "failed.");
+          }
+          LOG.info("Hdds Datanode login successful.");
+        }
         datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
         startPlugins();
         // Starting HDDS Daemons
         datanodeStateMachine.startDaemon();
       } catch (IOException e) {
         throw new RuntimeException("Can't start the HDDS datanode plugin", e);
+      } catch (AuthenticationException ex) {
+        throw new RuntimeException("Fail to authentication when starting" +
+            " HDDS datanode plugin", ex);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 41dd89a..dce2a45 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
   }
 
   @Override protected String getKeytabFile() {
-    return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+    return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
   }
 
   @Override protected String getSpnegoPrincipal() {
-    return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+    return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
   }
 
   @Override protected String getEnabledKey() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
deleted file mode 100644
index 199d52c..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
deleted file mode 100644
index db211bc..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
-   ozone.kdc:
-      image: ahadoop/kdc:v1
-   namenode:
-      image: ahadoop/ozone:v1
-      hostname: namenode
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9000:9000
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
-   datanode:
-      image: ahadoop/ozone:v1
-      hostname: datanode
-      volumes:
-        - ${OZONEDIR}:/opt/hadoop
-      ports:
-        - 9874
-      env_file:
-        - ./docker-config
-      command: ["/opt/hadoop/bin/ozone","datanode"]
-   om:
-      image: ahadoop/ozone:v1
-      hostname: om
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9874:9874
-      environment:
-         ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
-      env_file:
-          - ./docker-config
-      command: ["/opt/hadoop/bin/ozone","ksm"]
-   scm:
-      image: ahadoop/ozone:v1
-      hostname: scm
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9876:9876
-      env_file:
-          - ./docker-config
-      environment:
-          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
deleted file mode 100644
index cdf72ce..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.ksm.address=om
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_hdds.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=ozone.kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=10
-KERBEROS_ENABLED=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 0119eb5..303ebd9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -91,8 +91,8 @@ public final class OMConfigKeys {
       + "kerberos.keytab.file";
   public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
       + ".kerberos.principal";
-  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+  public static final String OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE =
       "ozone.om.http.kerberos.keytab.file";
-  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+  public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
       = "ozone.om.http.kerberos.principal";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 14c6800..5791de9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -117,9 +117,9 @@ public final class TestSecureOzoneCluster {
         conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
      createPrincipal(spnegoKeytab,
          conf.get(ScmConfigKeys
-             .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+             .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
         conf.get(OMConfigKeys
-            .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+            .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
     createPrincipal(omKeyTab,
         conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
   }
@@ -153,12 +153,12 @@ public final class TestSecureOzoneCluster {
 
     conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
         "scm/" + host + "@" + realm);
-    conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY,
         "HTTP_SCM/" + host + "@" + realm);
 
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
         "om/" + host + "@" + realm);
-    conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+    conf.set(OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY,
         "HTTP_OM/" + host + "@" + realm);
 
     scmKeytab = new File(workDir, "scm.keytab");
@@ -167,7 +167,8 @@ public final class TestSecureOzoneCluster {
 
     conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
         scmKeytab.getAbsolutePath());
-    conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+    conf.set(
+        ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
         spnegoKeytab.getAbsolutePath());
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
         omKeyTab.getAbsolutePath());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index be12fa8..8f7f058 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -64,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
   }
 
   @Override protected String getKeytabFile() {
-    return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
+    return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
   }
 
   @Override protected String getSpnegoPrincipal() {
-    return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+    return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
   }
 
   @Override protected String getEnabledKey() {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[44/50] [abbrv] hadoop git commit: HDDS-778. Add an interface for CA and Clients for Certificate operations Contributed by Anu Engineer.

Posted by xy...@apache.org.
HDDS-778. Add an interface for CA and Clients for Certificate operations
Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4770e9de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4770e9de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4770e9de

Branch: refs/heads/HDDS-4
Commit: 4770e9dea8199753962c6517bff96fd39fbfd826
Parents: 8bbc95e
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Nov 8 09:54:27 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../authority/CertificateServer.java            |  99 ++++++++++++
 .../certificate/authority/package-info.java     |  22 +++
 .../certificate/client/CertificateClient.java   | 159 +++++++++++++++++++
 .../x509/certificate/client/package-info.java   |  22 +++
 4 files changed, 302 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
new file mode 100644
index 0000000..9332e5b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+
+import java.security.cert.X509Certificate;
+import java.util.concurrent.Future;
+
+/**
+ * Interface for Certificate Authority. This can be extended to talk to external
+ * CAs later or HSMs later.
+ */
+public interface CertificateServer {
+  /**
+   * Initialize the Certificate Authority.
+   *
+   * @param securityConfig - Security Configuration.
+   * @param type - The Type of CertificateServer we are creating, we make this
+   * explicit so that when we read code it is visible to the users.
+   * @throws SCMSecurityException - Throws if the init fails.
+   */
+  void init(SecurityConfig securityConfig, CAType type)
+      throws SCMSecurityException;
+
+  /**
+   * Returns the CA Certificate for this CA.
+   *
+   * @return X509CertificateHolder - Certificate for this CA.
+   * @throws SCMSecurityException -- usually thrown if this CA is not
+   *                              initialized.
+   */
+  X509CertificateHolder getCACertificate()
+      throws SCMSecurityException;
+
+  /**
+   * Request a Certificate based on Certificate Signing Request.
+   *
+   * @param csr - Certificate Signing Request.
+   * @return A future that will have this certificate when this request is
+   * approved.
+   * @throws SCMSecurityException - on Error.
+   */
+  Future<X509CertificateHolder> requestCertificate(CertificateSignRequest csr,
+      CertificateApprover approver) throws SCMSecurityException;
+
+  /**
+   * Revokes a Certificate issued by this CertificateServer.
+   *
+   * @param certificate - Certificate to revoke
+   * @param approver - Approval process to follow.
+   * @return Future that tells us what happened.
+   * @throws SCMSecurityException - on Error.
+   */
+  Future<Boolean> revokeCertificate(X509Certificate certificate,
+      CertificateApprover approver) throws SCMSecurityException;
+
+  /**
+   * TODO : CRL, OCSP etc. Later. This is the start of a CertificateServer
+   * framework.
+   */
+
+  /**
+   * Approval Types for a certificate request.
+   */
+  enum CertificateApprover {
+    KERBEROS_TRUSTED, /* The Request came from a DN using Kerberos Identity*/
+    MANUAL, /* Wait for a Human being to approve this certificate */
+    TESTING_AUTOMATIC /* For testing purpose, Automatic Approval. */
+  }
+
+  /**
+   * Make it explicit what type of CertificateServer we are creating here.
+   */
+  enum CAType {
+    SELF_SIGNED_CA,
+    INTERMEDIARY_CA
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
new file mode 100644
index 0000000..af53904
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to Certificate Life Cycle or Certificate Authority Server.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
new file mode 100644
index 0000000..1b6f576
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.client;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+
+import java.io.InputStream;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.cert.CertStore;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
+/**
+ * Certificate client provides and interface to certificate operations that
+ * needs to be performed by all clients in the Ozone eco-system.
+ */
+public interface CertificateClient {
+
+  /**
+   * Returns the private key of the specified component if it exists on the
+   * local system.
+   *
+   * @param component - String name like DN, OM, SCM etc.
+   * @return private key or Null if there is no data.
+   */
+  PrivateKey getPrivateKey(String component);
+
+  /**
+   * Returns the public key of the specified component if it exists on the local
+   * system.
+   *
+   * @param component - String name like DN, OM, SCM etc.
+   * @return public key or Null if there is no data.
+   */
+  PublicKey getPublicKey(String component);
+
+  /**
+   * Returns the certificate  of the specified component if it exists on the
+   * local system.
+   *
+   * @param component - String name like DN, OM, SCM etc.
+   * @return certificate or Null if there is no data.
+   */
+  X509Certificate getCertificate(String component);
+
+  /**
+   * Verifies if this certificate is part of a trusted chain.
+   *
+   * @return true if it trusted, false otherwise.
+   */
+  boolean verifyCertificate(X509Certificate certificate);
+
+  /**
+   * Creates digital signature over the data stream using the components private
+   * key.
+   *
+   * @param stream - Data stream to sign.
+   * @return byte array - containing the signature.
+   */
+  byte[] signDataStream(InputStream stream, String component)
+      throws CertificateException;
+
+  /**
+   * Verifies a digital Signature, given the signature and the certificate of
+   * the signer.
+   * @param stream - Data Stream.
+   * @param signature - Byte Array containing the signature.
+   * @param cert - Certificate of the Signer.
+   * @return true if verified, false if not.
+   */
+  boolean verifySignature(InputStream stream, byte[] signature,
+      X509Certificate cert);
+
+  /**
+   * Returns a CSR builder that can be used to creates a Certificate sigining
+   * request.
+   *
+   * @return CertificateSignRequest.Builder
+   */
+  CertificateSignRequest.Builder getCSRBuilder();
+
+  /**
+   * Get the certificate of well-known entity from SCM.
+   *
+   * @param query - String Query, please see the implementation for the
+   * discussion on the query formats.
+   * @return X509Certificate or null if not found.
+   */
+  X509Certificate queryCertificate(String query);
+
+  /**
+   * Stores the private key of a specified component.
+   *
+   * @param key - private key
+   * @param component - name of the component.
+   * @throws CertificateException
+   */
+  void storePrivateKey(PrivateKey key, String component)
+      throws CertificateException;
+
+  /**
+   * Stores the public key of a specified component.
+   *
+   * @param key - public key
+   * @throws CertificateException
+   */
+  void storePublicKey(PublicKey key, String component)
+      throws CertificateException;
+
+  /**
+   * Stores the Certificate of a specific component.
+   *
+   * @param certificate - X509 Certificate
+   * @param component - Name of the component.
+   * @throws CertificateException
+   */
+  void storeCertificate(X509Certificate certificate, String component)
+      throws CertificateException;
+
+  /**
+   * Stores the trusted chain of certificates for a specific component.
+   *
+   * @param certStore - Cert Store.
+   * @param component - Trust Chain.
+   * @throws CertificateException
+   */
+  void storeTrustChain(CertStore certStore,
+      String component) throws CertificateException;
+
+  /**
+   * Stores the trusted chain of certificates for a specific component.
+   *
+   * @param certificates - List of Certificates.
+   * @param component - String component.
+   * @throws CertificateException
+   */
+  void storeTrustChain(List<X509Certificate> certificates,
+      String component) throws CertificateException;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
new file mode 100644
index 0000000..dea609b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to creating and using certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.client;
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[36/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager Delegation Token support. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-8. Add OzoneManager Delegation Token support. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59767be1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59767be1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59767be1

Branch: refs/heads/HDDS-4
Commit: 59767be13a37557b6d51cd5356c58ebdfad63896
Parents: 9c0baf0
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 15 12:18:19 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |   4 +
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   1 +
 .../common/src/main/resources/ozone-default.xml |  33 +
 .../ozone/client/protocol/ClientProtocol.java   |  32 +
 .../hadoop/ozone/client/rest/RestClient.java    |  41 ++
 .../hadoop/ozone/client/rpc/RpcClient.java      |  41 ++
 .../ozone/client/TestHddsClientUtils.java       |   2 +-
 .../java/org/apache/hadoop/ozone/OmUtils.java   |  17 +-
 .../apache/hadoop/ozone/om/OMConfigKeys.java    |  13 +
 .../ozone/om/protocol/OzoneManagerProtocol.java |   2 +-
 .../protocol/OzoneManagerSecurityProtocol.java  |  67 +++
 ...neManagerProtocolClientSideTranslatorPB.java |  77 +++
 .../om/protocolPB/OzoneManagerProtocolPB.java   |   3 +
 .../hadoop/ozone/protocolPB/OMPBHelper.java     |  38 ++
 .../security/OzoneDelegationTokenSelector.java  |   3 +-
 .../ozone/security/OzoneSecretManager.java      | 598 +++++++++++++++++++
 .../hadoop/ozone/security/OzoneSecretStore.java | 250 ++++++++
 .../ozone/security/OzoneSecurityException.java  | 104 ++++
 .../src/main/proto/OzoneManagerProtocol.proto   |  19 +
 .../ozone/security/TestOzoneSecretManager.java  | 216 +++++++
 .../hadoop/ozone/TestSecureOzoneCluster.java    | 309 ++++++++--
 .../apache/hadoop/ozone/om/OzoneManager.java    | 300 ++++++++--
 ...neManagerProtocolServerSideTranslatorPB.java |  59 ++
 23 files changed, 2150 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index ac97422..742fe3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -332,6 +332,10 @@ public final class OzoneConfigKeys {
   public static final String OZONE_CONTAINER_COPY_WORKDIR =
       "hdds.datanode.replication.work.dir";
 
+  public static final String OZONE_MAX_KEY_LEN =
+      "ozone.max.key.len";
+  public static final int OZONE_MAX_KEY_LEN_DEFAULT = 1024 * 1024;
+
   /**
    * Config properties to set client side checksum properties.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 096baee..e7c354c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -99,6 +99,7 @@ public final class OzoneConsts {
   public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
   public static final String DELETED_BLOCK_DB = "deletedBlock.db";
   public static final String OM_DB_NAME = "om.db";
+  public static final String OZONE_MANAGER_TOKEN_DB_NAME = "om-token.db";
 
   public static final String STORAGE_DIR_CHUNKS = "chunks";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5fb594b..766f31d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -984,6 +984,15 @@
       every principal specified in the keytab file.
     </description>
   </property>
+  <property>
+    <name>ozone.max.key.len</name>
+    <value>1048576</value>
+    <tag>OZONE, SECURITY</tag>
+    <description>
+      Maximum length of private key in Ozone. Used in Ozone delegation and
+      block tokens.
+    </description>
+  </property>
 
   <!--Client Settings-->
   <property>
@@ -1492,4 +1501,28 @@
       Name of file which stores public key generated for SCM CA.
     </description>
   </property>
+  <property>
+    <name>ozone.manager.delegation.remover.scan.interval</name>
+    <value>3600000</value>
+    <description>
+      Time interval after which ozone secret manger scans for expired
+      delegation token.
+    </description>
+  </property>
+  <property>
+    <name>ozone.manager.delegation.token.renew-interval</name>
+    <value>1d</value>
+    <description>
+      Default time interval after which ozone delegation token will
+      require renewal before any further use.
+    </description>
+  </property>
+  <property>
+    <name>ozone.manager.delegation.token.max-lifetime</name>
+    <value>7d</value>
+    <description>
+      Default max time interval after which ozone delegation token will
+      not be renewed.
+    </description>
+  </property>
 </configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 17634f2..2fe302e 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.protocol;
 
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.client.*;
 import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -30,7 +31,9 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 
 import java.io.IOException;
 import java.util.List;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
 
 /**
  * An implementer of this interface is capable of connecting to Ozone Cluster
@@ -388,4 +391,33 @@ public interface ClientProtocol {
    */
   void close() throws IOException;
 
+  /**
+   * Get a valid Delegation Token.
+   *
+   * @param renewer the designated renewer for the token
+   * @return Token<OzoneDelegationTokenSelector>
+   * @throws IOException
+   */
+  Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException;
+
+  /**
+   * Renew an existing delegation token.
+   *
+   * @param token delegation token obtained earlier
+   * @return the new expiration time
+   * @throws IOException
+   */
+  long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException;
+
+  /**
+   * Cancel an existing delegation token.
+   *
+   * @param token delegation token
+   * @throws IOException
+   */
+  void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException;
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 3fd0927..6e2841d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -44,10 +45,12 @@ import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.ozone.web.response.ListBuckets;
 import org.apache.hadoop.ozone.web.response.ListKeys;
 import org.apache.hadoop.ozone.web.response.ListVolumes;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Time;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
@@ -667,6 +670,44 @@ public class RestClient implements ClientProtocol {
     }
   }
 
+  /**
+   * Get a valid Delegation Token. Not supported for RestClient.
+   *
+   * @param renewer the designated renewer for the token
+   * @return Token<OzoneDelegationTokenSelector>
+   * @throws IOException
+   */
+  @Override
+  public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException {
+    throw new IOException("Method not supported");
+  }
+
+  /**
+   * Renew an existing delegation token. Not supported for RestClient.
+   *
+   * @param token delegation token obtained earlier
+   * @return the new expiration time
+   * @throws IOException
+   */
+  @Override
+  public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    throw new IOException("Method not supported");
+  }
+
+  /**
+   * Cancel an existing delegation token. Not supported for RestClient.
+   *
+   * @param token delegation token
+   * @throws IOException
+   */
+  @Override
+  public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    throw new IOException("Method not supported");
+  }
+
   @Override
   public OzoneInputStream getKey(
       String volumeName, String bucketName, String keyName)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 1b70f3b..3575f01 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -67,7 +67,10 @@ import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
 import org.apache.logging.log4j.util.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -404,6 +407,44 @@ public class RpcClient implements ClientProtocol {
     ozoneManagerClient.setBucketProperty(builder.build());
   }
 
+  /**
+   * Get a valid Delegation Token.
+   *
+   * @param renewer the designated renewer for the token
+   * @return Token<OzoneDelegationTokenSelector>
+   * @throws IOException
+   */
+  @Override
+  public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException {
+    return ozoneManagerClient.getDelegationToken(renewer);
+  }
+
+  /**
+   * Renew an existing delegation token.
+   *
+   * @param token delegation token obtained earlier
+   * @return the new expiration time
+   * @throws IOException
+   */
+  @Override
+  public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    return ozoneManagerClient.renewDelegationToken(token);
+  }
+
+  /**
+   * Cancel an existing delegation token.
+   *
+   * @param token delegation token
+   * @throws IOException
+   */
+  @Override
+  public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    ozoneManagerClient.cancelDelegationToken(token);
+  }
+
   @Override
   public void setBucketVersioning(
       String volumeName, String bucketName, Boolean versioning)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
index 9e60e4e..ff4aeb3 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
@@ -84,7 +84,7 @@ public class TestHddsClientUtils {
   }
 
   @Test
-  public void testGetOmAddress() {
+  public void testgetOmSocketAddress() {
     final Configuration conf = new OzoneConfiguration();
 
     // First try a client address with just a host name. Verify it falls

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 9280648..e7ed84f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -54,14 +54,21 @@ public final class OmUtils {
    * @param conf
    * @return Target InetSocketAddress for the SCM service endpoint.
    */
-  public static InetSocketAddress getOmAddress(
-      Configuration conf) {
+  public static InetSocketAddress getOmAddress(Configuration conf) {
+    return NetUtils.createSocketAddr(getOmRpcAddress(conf));
+  }
+
+  /**
+   * Retrieve the socket address that is used by OM.
+   * @param conf
+   * @return Target InetSocketAddress for the SCM service endpoint.
+   */
+  public static String getOmRpcAddress(Configuration conf) {
     final Optional<String> host = getHostNameFromConfigKeys(conf,
         OZONE_OM_ADDRESS_KEY);
 
-    return NetUtils.createSocketAddr(
-        host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
-            getOmRpcPort(conf));
+    return host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
+        getOmRpcPort(conf);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 303ebd9..de24184 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -95,4 +95,17 @@ public final class OMConfigKeys {
       "ozone.om.http.kerberos.keytab.file";
   public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
       = "ozone.om.http.kerberos.principal";
+  // Delegation token related keys
+  public static final String  DELEGATION_REMOVER_SCAN_INTERVAL_KEY =
+      "ozone.manager.delegation.remover.scan.interval";
+  public static final long    DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT =
+      60*60*1000;
+  public static final String  DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
+      "ozone.manager.delegation.token.renew-interval";
+  public static final long    DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
+      24*60*60*1000;  // 1 day = 86400000 ms
+  public static final String  DELEGATION_TOKEN_MAX_LIFETIME_KEY =
+      "ozone.manager.delegation.token.max-lifetime";
+  public static final long    DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
+      7*24*60*60*1000; // 7 days
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 233666a..06933d2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.KerberosInfo;
  */
 @KerberosInfo(
     serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
-public interface OzoneManagerProtocol {
+public interface OzoneManagerProtocol  extends OzoneManagerSecurityProtocol {
 
   /**
    * Creates a volume.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
new file mode 100644
index 0000000..15873d0
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * Security protocol for a secure OzoneManager.
+ */
+@KerberosInfo(
+    serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+public interface OzoneManagerSecurityProtocol {
+
+  /**
+   * Get a valid Delegation Token.
+   *
+   * @param renewer the designated renewer for the token
+   * @return Token<OzoneDelegationTokenSelector>
+   * @throws IOException
+   */
+  @Idempotent
+  Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException;
+
+  /**
+   * Renew an existing delegation token.
+   *
+   * @param token delegation token obtained earlier
+   * @return the new expiration time
+   * @throws IOException
+   */
+  @Idempotent
+  long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException;
+
+  /**
+   * Cancel an existing delegation token.
+   *
+   * @param token delegation token
+   * @throws IOException
+   */
+  @Idempotent
+  void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 6c8c932..85ae4a8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolTranslator;
 import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
@@ -139,6 +140,13 @@ import java.io.IOException;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.stream.Collectors;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.token.Token;
 
 /**
  *  The client side implementation of OzoneManagerProtocol.
@@ -884,4 +892,73 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
   public Object getUnderlyingProxyObject() {
     return null;
   }
+
+  /**
+   * Get a valid Delegation Token.
+   *
+   * @param renewer the designated renewer for the token
+   * @return Token<OzoneDelegationTokenSelector>
+   * @throws IOException
+   */
+  @Override
+  public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+      throws IOException {
+    GetDelegationTokenRequestProto req = GetDelegationTokenRequestProto
+        .newBuilder()
+        .setRenewer(renewer == null ? "" : renewer.toString())
+        .build();
+    try {
+      GetDelegationTokenResponseProto resp =
+          rpcProxy.getDelegationToken(NULL_RPC_CONTROLLER, req);
+      return resp.hasToken() ?
+          OMPBHelper.convertToDelegationToken(resp.getToken()) : null;
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    } catch (Exception e){
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  /**
+   * Renew an existing delegation token.
+   *
+   * @param token delegation token obtained earlier
+   * @return the new expiration time
+   * @throws IOException
+   */
+  @Override
+  public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    RenewDelegationTokenRequestProto req =
+        RenewDelegationTokenRequestProto.newBuilder().
+            setToken(OMPBHelper.convertToTokenProto(token)).
+            build();
+    try {
+      return rpcProxy.renewDelegationToken(NULL_RPC_CONTROLLER, req)
+          .getNewExpiryTime();
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  /**
+   * Cancel an existing delegation token.
+   *
+   * @param token delegation token
+   * @throws IOException
+   */
+  @Override
+  public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+      throws IOException {
+    CancelDelegationTokenRequestProto req = CancelDelegationTokenRequestProto
+        .newBuilder()
+        .setToken(OMPBHelper.convertToTokenProto(token))
+        .build();
+    try {
+      rpcProxy.cancelDelegationToken(NULL_RPC_CONTROLLER, req);
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index 27e8f22..175527b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneManagerService;
 import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
 
 /**
  * Protocol used to communicate with OM.
@@ -32,6 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
     protocolVersion = 1)
 @KerberosInfo(
     serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+@TokenInfo(OzoneDelegationTokenSelector.class)
 @InterfaceAudience.Private
 public interface OzoneManagerProtocolPB
     extends OzoneManagerService.BlockingInterface {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
index d57d32e..df069ce 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.ozone.protocolPB;
 
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneAclInfo;
@@ -24,6 +26,9 @@ import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.security.token.Token;
 
 /**
  * Utilities for converting protobuf classes.
@@ -110,4 +115,37 @@ public final class OMPBHelper {
 
     return new OzoneAcl(aclType, aclInfo.getName(), aclRights);
   }
+
+  /**
+   * Converts Ozone delegation token to @{@link TokenProto}.
+   * @return tokenProto
+   */
+  public static TokenProto convertToTokenProto(Token<?> tok) {
+    if(tok == null){
+      throw new IllegalArgumentException("Invalid argument: token is null");
+    }
+
+    return TokenProto.newBuilder().
+        setIdentifier(getByteString(tok.getIdentifier())).
+        setPassword(getByteString(tok.getPassword())).
+        setKind(tok.getKind().toString()).
+        setService(tok.getService().toString()).build();
+  }
+
+  public static ByteString getByteString(byte[] bytes) {
+    // return singleton to reduce object allocation
+    return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
+  }
+
+  /**
+   * Converts @{@link TokenProto} to Ozone delegation token.
+   *
+   * @return Ozone
+   */
+  public static Token<OzoneTokenIdentifier> convertToDelegationToken(
+      TokenProto tokenProto) {
+    return new Token<>(tokenProto.getIdentifier()
+        .toByteArray(), tokenProto.getPassword().toByteArray(), new Text(
+        tokenProto.getKind()), new Text(tokenProto.getService()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
index 0d63c4e..8e35f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.security;
 
 import java.util.Collection;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -32,7 +31,7 @@ import org.slf4j.LoggerFactory;
  */
 @InterfaceAudience.Private
 public class OzoneDelegationTokenSelector
-    extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+    extends AbstractDelegationTokenSelector<OzoneTokenIdentifier> {
 
   public OzoneDelegationTokenSelector() {
     super(OzoneTokenIdentifier.KIND_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
new file mode 100644
index 0000000..0c84404
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
@@ -0,0 +1,598 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SecretManager for Ozone Master. Responsible for signing identifiers with
+ * private key,
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretManager<T extends OzoneTokenIdentifier>
+    extends SecretManager<T> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneSecretManager.class);
+  /**
+   * The name of the Private/Public Key based hashing algorithm.
+   */
+  private static final String DEFAULT_SIGNATURE_ALGORITHM = "SHA256withRSA";
+  private final long tokenMaxLifetime;
+  private final long tokenRenewInterval;
+  private final long tokenRemoverScanInterval;
+  private final Text service;
+  private final Map<Integer, OzoneSecretKey> allKeys;
+  private final Map<T, TokenInfo> currentTokens;
+  private final OzoneSecretStore store;
+  private Thread tokenRemoverThread;
+  private volatile boolean running;
+  private AtomicInteger tokenSequenceNumber;
+  private OzoneSecretKey currentKey;
+  private AtomicInteger currentKeyId;
+  /**
+   * If the delegation token update thread holds this lock, it will not get
+   * interrupted.
+   */
+  private Object noInterruptsLock = new Object();
+  private int maxKeyLength;
+
+  /**
+   * Create a secret manager.
+   *
+   * @param conf configuration.
+   * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
+   * milliseconds
+   * @param tokenRenewInterval how often the tokens must be renewed in
+   * milliseconds
+   * @param dtRemoverScanInterval how often the tokens are scanned for expired
+   * tokens in milliseconds
+   */
+  public OzoneSecretManager(OzoneConfiguration conf, long tokenMaxLifetime,
+      long tokenRenewInterval, long dtRemoverScanInterval, Text service)
+      throws IOException {
+    this.tokenMaxLifetime = tokenMaxLifetime;
+    this.tokenRenewInterval = tokenRenewInterval;
+    this.tokenRemoverScanInterval = dtRemoverScanInterval;
+
+    currentTokens = new ConcurrentHashMap();
+    allKeys = new ConcurrentHashMap<>();
+    currentKeyId = new AtomicInteger();
+    tokenSequenceNumber = new AtomicInteger();
+    this.store = new OzoneSecretStore(conf);
+    loadTokenSecretState(store.loadState());
+    this.service = service;
+    this.maxKeyLength = conf.getInt(OzoneConfigKeys.OZONE_MAX_KEY_LEN,
+        OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
+  }
+
+  @Override
+  public T createIdentifier() {
+    return (T) T.newInstance();
+  }
+
+  /**
+   * Create new Identifier with given,owner,renwer and realUser.
+   *
+   * @return T
+   */
+  public T createIdentifier(Text owner, Text renewer, Text realUser) {
+    return (T) T.newInstance(owner, renewer, realUser);
+  }
+
+  /**
+   * Returns {@link Token} for given identifier.
+   *
+   * @param owner
+   * @param renewer
+   * @param realUser
+   * @return Token
+   * @throws IOException to allow future exceptions to be added without breaking
+   *                     compatibility
+   */
+  public Token<T> createToken(Text owner, Text renewer, Text realUser)
+      throws IOException {
+    T identifier = createIdentifier(owner, renewer, realUser);
+    updateIdentifierDetails(identifier);
+
+    byte[] password = createPassword(identifier.getBytes(),
+        currentKey.getPrivateKey());
+    addToTokenStore(identifier, password);
+    Token<T> token = new Token<>(identifier.getBytes(), password,
+        identifier.getKind(), service);
+    if (LOG.isTraceEnabled()) {
+      long expiryTime = identifier.getIssueDate() + tokenRenewInterval;
+      String tokenId = identifier.toStringStable();
+      LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+          expiryTime, tokenId);
+    }
+
+    return token;
+  }
+
+  /**
+   * Stores given identifier in token store.
+   *
+   * @param identifier
+   * @param password
+   * @throws IOException
+   */
+  private void addToTokenStore(T identifier, byte[] password)
+      throws IOException {
+    TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
+        + tokenRenewInterval, password, identifier.getTrackingId());
+    currentTokens.put(identifier, tokenInfo);
+    store.storeToken(identifier, tokenInfo.getRenewDate());
+  }
+
+  /**
+   * Updates issue date, master key id and sequence number for identifier.
+   *
+   * @param identifier the identifier to validate
+   */
+  private void updateIdentifierDetails(T identifier) {
+    int sequenceNum;
+    long now = Time.monotonicNow();
+    sequenceNum = incrementDelegationTokenSeqNum();
+    identifier.setIssueDate(now);
+    identifier.setMasterKeyId(currentKey.getKeyId());
+    identifier.setSequenceNumber(sequenceNum);
+    identifier.setMaxDate(Time.monotonicNow() + tokenMaxLifetime);
+  }
+
+  /**
+   * Compute HMAC of the identifier using the private key and return the output
+   * as password.
+   *
+   * @param identifier
+   * @param privateKey
+   * @return byte[] signed byte array
+   */
+  public byte[] createPassword(byte[] identifier, PrivateKey privateKey)
+      throws OzoneSecurityException {
+    try {
+      Signature rsaSignature = Signature.getInstance(
+          DEFAULT_SIGNATURE_ALGORITHM);
+      rsaSignature.initSign(privateKey);
+      rsaSignature.update(identifier);
+      return rsaSignature.sign();
+    } catch (InvalidKeyException | NoSuchAlgorithmException |
+        SignatureException ex) {
+      throw new OzoneSecurityException("Error while creating HMAC hash for " +
+          "token.", ex, OzoneSecurityException.ResultCodes
+          .SECRET_MANAGER_HMAC_ERROR);
+    }
+  }
+
+  @Override
+  public byte[] createPassword(T identifier) {
+    LOG.debug("Creating password for identifier: {}, currentKey: {}",
+        formatTokenId(identifier), currentKey.getKeyId());
+    byte[] password = null;
+    try {
+      password = createPassword(identifier.getBytes(),
+          currentKey.getPrivateKey());
+    } catch (IOException ioe) {
+      LOG.error("Could not store token {}!!", formatTokenId(identifier),
+          ioe);
+    }
+    return password;
+  }
+
+  @Override
+  public byte[] retrievePassword(T identifier) throws InvalidToken {
+    return checkToken(identifier).getPassword();
+  }
+
+  /**
+   * Renew a delegation token.
+   *
+   * @param token the token to renew
+   * @param renewer the full principal name of the user doing the renewal
+   * @return the new expiration time
+   * @throws InvalidToken           if the token is invalid
+   * @throws AccessControlException if the user can't renew token
+   */
+  public synchronized long renewToken(Token<T> token, String renewer)
+      throws IOException {
+    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+    DataInputStream in = new DataInputStream(buf);
+    T id = (T) T.readProtoBuf(in);
+    LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
+        formatTokenId(id), currentTokens.size());
+
+    long now = Time.monotonicNow();
+    if (id.getMaxDate() < now) {
+      throw new InvalidToken(renewer + " tried to renew an expired token "
+          + formatTokenId(id) + " max expiration date: "
+          + Time.formatTime(id.getMaxDate())
+          + " currentTime: " + Time.formatTime(now));
+    }
+    checkToken(id);
+    if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
+      throw new AccessControlException(renewer +
+          " tried to renew a token " + formatTokenId(id)
+          + " without a renewer");
+    }
+    if (!id.getRenewer().toString().equals(renewer)) {
+      throw new AccessControlException(renewer
+          + " tries to renew a token " + formatTokenId(id)
+          + " with non-matching renewer " + id.getRenewer());
+    }
+    OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
+    if (key == null) {
+      throw new InvalidToken("Unable to find master key for keyId="
+          + id.getMasterKeyId()
+          + " from cache. Failed to renew an unexpired token "
+          + formatTokenId(id) + " with sequenceNumber="
+          + id.getSequenceNumber());
+    }
+    byte[] password = createPassword(token.getIdentifier(),
+        key.getPrivateKey());
+
+    long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
+    try {
+      addToTokenStore(id, password);
+    } catch (IOException e) {
+      LOG.error("Unable to update token " + id.getSequenceNumber(), e);
+    }
+    return renewTime;
+  }
+
+  /**
+   * Cancel a token by removing it from store and cache.
+   *
+   * @return Identifier of the canceled token
+   * @throws InvalidToken           for invalid token
+   * @throws AccessControlException if the user isn't allowed to cancel
+   */
+  public T cancelToken(Token<T> token, String canceller) throws IOException {
+    T id = (T) T.readProtoBuf(token.getIdentifier());
+    LOG.debug("Token cancellation requested for identifier: {}",
+        formatTokenId(id));
+
+    if (id.getUser() == null) {
+      throw new InvalidToken("Token with no owner " + formatTokenId(id));
+    }
+    String owner = id.getUser().getUserName();
+    Text renewer = id.getRenewer();
+    HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
+    String cancelerShortName = cancelerKrbName.getShortName();
+    if (!canceller.equals(owner)
+        && (renewer == null || renewer.toString().isEmpty()
+        || !cancelerShortName
+        .equals(renewer.toString()))) {
+      throw new AccessControlException(canceller
+          + " is not authorized to cancel the token " + formatTokenId(id));
+    }
+    try {
+      store.removeToken(id);
+    } catch (IOException e) {
+      LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
+    }
+    TokenInfo info = currentTokens.remove(id);
+    if (info == null) {
+      throw new InvalidToken("Token not found " + formatTokenId(id));
+    }
+    return id;
+  }
+
+  public int getCurrentKeyId() {
+    return currentKeyId.get();
+  }
+
+  public void setCurrentKeyId(int keyId) {
+    currentKeyId.set(keyId);
+  }
+
+  public int incrementCurrentKeyId() {
+    return currentKeyId.incrementAndGet();
+  }
+
+  public int getDelegationTokenSeqNum() {
+    return tokenSequenceNumber.get();
+  }
+
+  public void setDelegationTokenSeqNum(int seqNum) {
+    tokenSequenceNumber.set(seqNum);
+  }
+
+  public int incrementDelegationTokenSeqNum() {
+    return tokenSequenceNumber.incrementAndGet();
+  }
+
+  /**
+   * Validates if given token is valid.
+   *
+   * @param identifier
+   * @param password
+   */
+  private boolean validateToken(T identifier, byte[] password) {
+    try {
+      Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+      rsaSignature.initVerify(currentKey.getPublicKey());
+      rsaSignature.update(identifier.getBytes());
+      return rsaSignature.verify(password);
+    } catch (NoSuchAlgorithmException | SignatureException |
+        InvalidKeyException e) {
+      return false;
+    }
+  }
+
+  /**
+   * Checks if TokenInfo for the given identifier exists in database and if the
+   * token is expired.
+   */
+  public TokenInfo checkToken(T identifier) throws InvalidToken {
+    TokenInfo info = currentTokens.get(identifier);
+    if (info == null) {
+      throw new InvalidToken("token " + formatTokenId(identifier)
+          + " can't be found in cache");
+    }
+    long now = Time.monotonicNow();
+    if (info.getRenewDate() < now) {
+      throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+          "expired, current time: " + Time.formatTime(now) +
+          " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+    }
+    if (!validateToken(identifier, info.getPassword())) {
+      throw new InvalidToken("Tampared/Inavalid token.");
+    }
+    return info;
+  }
+
+  // TODO: handle roll private key/certificate
+  private synchronized void removeExpiredKeys() {
+    long now = Time.monotonicNow();
+    for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+        .iterator(); it.hasNext();) {
+      Map.Entry<Integer, OzoneSecretKey> e = it.next();
+      OzoneSecretKey key = e.getValue();
+      if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
+        if (!key.equals(currentKey)) {
+          it.remove();
+          try {
+            store.removeTokenMasterKey(key);
+          } catch (IOException ex) {
+            LOG.error("Unable to remove master key " + key.getKeyId(), ex);
+          }
+        }
+      }
+    }
+  }
+
+  private void loadTokenSecretState(OzoneManagerSecretState<T> state)
+      throws IOException {
+    LOG.info("Loading token state into token manager.");
+    for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
+      allKeys.putIfAbsent(key.getKeyId(), key);
+    }
+    for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
+      addPersistedDelegationToken(entry.getKey(), entry.getValue());
+    }
+  }
+
+  private String formatTokenId(T id) {
+    return "(" + id + ")";
+  }
+
+  private void addPersistedDelegationToken(
+      T identifier, long renewDate)
+      throws IOException {
+    if (running) {
+      // a safety check
+      throw new IOException(
+          "Can't add persisted delegation token to a running SecretManager.");
+    }
+    int keyId = identifier.getMasterKeyId();
+    OzoneSecretKey dKey = allKeys.get(keyId);
+    if (dKey == null) {
+      LOG.warn("No KEY found for persisted identifier "
+          + formatTokenId(identifier));
+      return;
+    }
+
+    PrivateKey privateKey = dKey.getPrivateKey();
+    byte[] password = createPassword(identifier.getBytes(), privateKey);
+    if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
+      setDelegationTokenSeqNum(identifier.getSequenceNumber());
+    }
+    if (currentTokens.get(identifier) == null) {
+      currentTokens.put(identifier, new TokenInfo(renewDate,
+          password, identifier.getTrackingId()));
+    } else {
+      throw new IOException("Same delegation token being added twice: "
+          + formatTokenId(identifier));
+    }
+  }
+
+  /**
+   * Should be called before this object is used.
+   */
+  public void startThreads(KeyPair keyPair) throws IOException {
+    Preconditions.checkState(!running);
+    updateCurrentKey(keyPair);
+    removeExpiredKeys();
+    synchronized (this) {
+      running = true;
+      tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
+      tokenRemoverThread.start();
+    }
+  }
+
+  public void stopThreads() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Stopping expired delegation token remover thread");
+    }
+    running = false;
+
+    if (tokenRemoverThread != null) {
+      synchronized (noInterruptsLock) {
+        tokenRemoverThread.interrupt();
+      }
+      try {
+        tokenRemoverThread.join();
+      } catch (InterruptedException e) {
+        throw new RuntimeException(
+            "Unable to join on token removal thread", e);
+      }
+    }
+  }
+
+  /**
+   * Stops the OzoneSecretManager.
+   *
+   * @throws IOException
+   */
+  public void stop() throws IOException {
+    stopThreads();
+    if (this.store != null) {
+      this.store.close();
+    }
+  }
+
+  /**
+   * Update the current master key. This is called once by startThreads before
+   * tokenRemoverThread is created,
+   */
+  private void updateCurrentKey(KeyPair keyPair) throws IOException {
+    LOG.info("Updating the current master key for generating tokens");
+
+    // TODO: fix me based on the certificate expire time to set the key
+    // expire time.
+    int newCurrentId = incrementCurrentKeyId();
+    OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
+        keyPair, maxKeyLength);
+
+    store.storeTokenMasterKey(newKey);
+    if (!allKeys.containsKey(newKey.getKeyId())) {
+      allKeys.put(newKey.getKeyId(), newKey);
+    }
+
+    synchronized (this) {
+      currentKey = newKey;
+    }
+  }
+
+  /**
+   * Remove expired delegation tokens from cache and persisted store.
+   */
+  private void removeExpiredToken() throws IOException {
+    long now = Time.monotonicNow();
+    synchronized (this) {
+      Iterator<Map.Entry<T,
+          TokenInfo>> i = currentTokens.entrySet().iterator();
+      while (i.hasNext()) {
+        Map.Entry<T,
+            TokenInfo> entry = i.next();
+        long renewDate = entry.getValue().getRenewDate();
+        if (renewDate < now) {
+          i.remove();
+          store.removeToken(entry.getKey());
+        }
+      }
+    }
+  }
+
+  /**
+   * Is Secret Manager running.
+   *
+   * @return true if secret mgr is running
+   */
+  public synchronized boolean isRunning() {
+    return running;
+  }
+
+  /**
+   * Returns expiry time of a token given its identifier.
+   *
+   * @param dtId DelegationTokenIdentifier of a token
+   * @return Expiry time of the token
+   * @throws IOException
+   */
+  public long getTokenExpiryTime(T dtId)
+      throws IOException {
+    TokenInfo info = currentTokens.get(dtId);
+    if (info != null) {
+      return info.getRenewDate();
+    } else {
+      throw new IOException("No delegation token found for this identifier");
+    }
+  }
+
+  private class ExpiredTokenRemover extends Thread {
+    private long lastTokenCacheCleanup;
+
+    @Override
+    public void run() {
+      LOG.info("Starting expired delegation token remover thread, "
+          + "tokenRemoverScanInterval=" + tokenRemoverScanInterval
+          / (60 * 1000) + " min(s)");
+      try {
+        while (running) {
+          long now = Time.monotonicNow();
+          if (lastTokenCacheCleanup + tokenRemoverScanInterval
+              < now) {
+            removeExpiredToken();
+            lastTokenCacheCleanup = now;
+          }
+          try {
+            Thread.sleep(Math.min(5000,
+                tokenRemoverScanInterval)); // 5 seconds
+          } catch (InterruptedException ie) {
+            LOG.error("ExpiredTokenRemover received " + ie);
+          }
+        }
+      } catch (Throwable t) {
+        LOG.error("ExpiredTokenRemover thread received unexpected exception",
+            t);
+        Runtime.getRuntime().exit(-1);
+      }
+    }
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
new file mode 100644
index 0000000..6528bcf
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.utils.MetadataKeyFilters;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MANAGER_TOKEN_DB_NAME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_MB;
+
+/**
+ * SecretStore for Ozone Master.
+ */
+public class OzoneSecretStore<T extends OzoneTokenIdentifier>
+    implements Closeable {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneSecretStore.class);
+  private static final String TOKEN_MASTER_KEY_KEY_PREFIX = "tokens/key_";
+  private static final String TOKEN_STATE_KEY_PREFIX = "tokens/token_";
+
+  @Override
+  public void close() throws IOException {
+    if (store != null) {
+      store.close();
+    }
+  }
+
+
+  /**
+   * Support class to maintain state of OzoneSecretStore.
+   */
+  public static class OzoneManagerSecretState<T> {
+
+    private Map<T, Long> tokenState = new HashMap<>();
+    private Set<OzoneSecretKey> tokenMasterKeyState = new HashSet<>();
+
+    public Map<T, Long> getTokenState() {
+      return tokenState;
+    }
+
+    public Set<OzoneSecretKey> ozoneManagerSecretState() {
+      return tokenMasterKeyState;
+    }
+  }
+
+  private MetadataStore store;
+
+  public OzoneSecretStore(OzoneConfiguration conf)
+      throws IOException {
+    File metaDir = getOzoneMetaDirPath(conf);
+    final int cacheSize = conf.getInt(OZONE_OM_DB_CACHE_SIZE_MB,
+        OZONE_OM_DB_CACHE_SIZE_DEFAULT);
+    File omTokenDBFile = new File(metaDir.getPath(),
+        OZONE_MANAGER_TOKEN_DB_NAME);
+    this.store = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setDbFile(omTokenDBFile)
+        .setCacheSize(cacheSize * OzoneConsts.MB)
+        .build();
+  }
+
+  public OzoneManagerSecretState loadState() throws IOException {
+    OzoneManagerSecretState state = new OzoneManagerSecretState();
+    int numKeys = loadMasterKeys(state);
+    LOG.info("Loaded " + numKeys + " token master keys");
+    int numTokens = loadTokens(state);
+    LOG.info("Loaded " + numTokens + " tokens");
+    return state;
+  }
+
+  public void storeTokenMasterKey(OzoneSecretKey key) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing master key " + key.getKeyId());
+    }
+    ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+    DataOutputStream dataStream = new DataOutputStream(memStream);
+    try {
+      key.write(dataStream);
+      dataStream.close();
+      dataStream = null;
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, dataStream);
+    }
+    try {
+      byte[] dbKey = getMasterKeyDBKey(key);
+      store.put(dbKey, memStream.toByteArray());
+    } catch (IOException e) {
+      LOG.error("Unable to store master key " + key.getKeyId(), e);
+      throw e;
+    }
+  }
+
+
+  public void removeTokenMasterKey(OzoneSecretKey key)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Removing master key " + key.getKeyId());
+    }
+
+    byte[] dbKey = getMasterKeyDBKey(key);
+    try {
+      store.delete(dbKey);
+    } catch (IOException e) {
+      LOG.error("Unable to delete master key " + key.getKeyId(), e);
+      throw e;
+    }
+  }
+
+  public void storeToken(T tokenId, Long renewDate)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing token " + tokenId.getSequenceNumber());
+    }
+
+    ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+    DataOutputStream dataStream = new DataOutputStream(memStream);
+    try {
+      tokenId.write(dataStream);
+      dataStream.writeLong(renewDate);
+      dataStream.close();
+      dataStream = null;
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, dataStream);
+    }
+
+    byte[] dbKey = getTokenDBKey(tokenId);
+    try {
+      store.put(dbKey, memStream.toByteArray());
+    } catch (IOException e) {
+      LOG.error("Unable to store token " + tokenId.toString(), e);
+      throw e;
+    }
+  }
+
+  public void updateToken(T tokenId, Long renewDate)
+      throws IOException {
+    storeToken(tokenId, renewDate);
+  }
+
+  public void removeToken(T tokenId)
+      throws IOException {
+    byte[] dbKey = getTokenDBKey(tokenId);
+    try {
+      store.delete(dbKey);
+    } catch (IOException e) {
+      LOG.error("Unable to remove token " + tokenId.toString(), e);
+      throw e;
+    }
+  }
+
+  public int loadMasterKeys(OzoneManagerSecretState state) throws IOException {
+    MetadataKeyFilters.MetadataKeyFilter filter =
+        (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+            .startsWith(TOKEN_MASTER_KEY_KEY_PREFIX);
+    List<Map.Entry<byte[], byte[]>> kvs = store
+        .getRangeKVs(null, Integer.MAX_VALUE, filter);
+    kvs.forEach(entry -> {
+      try {
+        loadTokenMasterKey(state, entry.getValue());
+      } catch (IOException e) {
+        LOG.warn("Failed to load master key ",
+            DFSUtil.bytes2String(entry.getKey()), e);
+      }
+    });
+    return kvs.size();
+  }
+
+  private void loadTokenMasterKey(OzoneManagerSecretState state, byte[] data)
+      throws IOException {
+    OzoneSecretKey key = OzoneSecretKey.readProtoBuf(data);
+    state.tokenMasterKeyState.add(key);
+  }
+
+  public int loadTokens(OzoneManagerSecretState state) throws IOException {
+    MetadataKeyFilters.MetadataKeyFilter filter =
+        (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+            .startsWith(TOKEN_STATE_KEY_PREFIX);
+    List<Map.Entry<byte[], byte[]>> kvs =
+        store.getRangeKVs(null, Integer.MAX_VALUE, filter);
+    kvs.forEach(entry -> {
+      try {
+        loadToken(state, entry.getValue());
+      } catch (IOException e) {
+        LOG.warn("Failed to load token ",
+            DFSUtil.bytes2String(entry.getKey()), e);
+      }
+    });
+    return kvs.size();
+  }
+
+  private void loadToken(OzoneManagerSecretState state, byte[] data)
+      throws IOException {
+    long renewDate;
+    DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+    T tokenId = (T) T.readProtoBuf(in);
+    try {
+      tokenId.readFields(in);
+      renewDate = in.readLong();
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, in);
+    }
+    state.tokenState.put(tokenId, renewDate);
+  }
+
+  private byte[] getMasterKeyDBKey(OzoneSecretKey masterKey) {
+    return DFSUtil.string2Bytes(
+        TOKEN_MASTER_KEY_KEY_PREFIX + masterKey.getKeyId());
+  }
+
+  private byte[] getTokenDBKey(T tokenId) {
+    return DFSUtil.string2Bytes(
+        TOKEN_STATE_KEY_PREFIX + tokenId.getSequenceNumber());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
new file mode 100644
index 0000000..66533f3
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.IOException;
+
+/**
+ * Security exceptions thrown at Ozone layer.
+ */
+public class OzoneSecurityException extends IOException {
+  private final OzoneSecurityException.ResultCodes result;
+
+  /**
+   * Constructs an {@code IOException} with {@code null}
+   * as its error detail message.
+   */
+  public OzoneSecurityException(OzoneSecurityException.ResultCodes result) {
+    this.result = result;
+  }
+
+  /**
+   * Constructs an {@code IOException} with the specified detail message.
+   *
+   * @param message The detail message (which is saved for later retrieval by
+   * the
+   * {@link #getMessage()} method)
+   */
+  public OzoneSecurityException(String message,
+      OzoneSecurityException.ResultCodes result) {
+    super(message);
+    this.result = result;
+  }
+
+  /**
+   * Constructs an {@code IOException} with the specified detail message
+   * and cause.
+   * <p>
+   * <p> Note that the detail message associated with {@code cause} is
+   * <i>not</i> automatically incorporated into this exception's detail
+   * message.
+   *
+   * @param message The detail message (which is saved for later retrieval by
+   * the
+   * {@link #getMessage()} method)
+   * @param cause The cause (which is saved for later retrieval by the {@link
+   * #getCause()} method).  (A null value is permitted, and indicates that the
+   * cause is nonexistent or unknown.)
+   * @since 1.6
+   */
+  public OzoneSecurityException(String message, Throwable cause,
+                     OzoneSecurityException.ResultCodes result) {
+    super(message, cause);
+    this.result = result;
+  }
+
+  /**
+   * Constructs an {@code IOException} with the specified cause and a
+   * detail message of {@code (cause==null ? null : cause.toString())}
+   * (which typically contains the class and detail message of {@code cause}).
+   * This constructor is useful for IO exceptions that are little more
+   * than wrappers for other throwables.
+   *
+   * @param cause The cause (which is saved for later retrieval by the {@link
+   * #getCause()} method).  (A null value is permitted, and indicates that the
+   * cause is nonexistent or unknown.)
+   * @since 1.6
+   */
+  public OzoneSecurityException(Throwable cause,
+      OzoneSecurityException.ResultCodes result) {
+    super(cause);
+    this.result = result;
+  }
+
+  /**
+   * Returns resultCode.
+   * @return ResultCode
+   */
+  public OzoneSecurityException.ResultCodes getResult() {
+    return result;
+  }
+
+  /**
+   * Error codes to make it easy to decode these exceptions.
+   */
+  public enum ResultCodes {
+    OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST,
+    SECRET_MANAGER_HMAC_ERROR
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 5caebf5..9af8dd0 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -35,6 +35,7 @@ This is similar to Namenode for Ozone.
 */
 
 import "hdds.proto";
+import "Security.proto";
 
 enum Status {
     OK = 1;
@@ -544,6 +545,24 @@ service OzoneManagerService {
     rpc getServiceList(ServiceListRequest)
     returns(ServiceListResponse);
 
+
+    /**
+     Returns delegation token signed by OzoneManager.
+    */
+    rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto)
+      returns(hadoop.common.GetDelegationTokenResponseProto);
+
+    /**
+     Renew delegation token signed by OzoneManager.
+    */
+    rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto)
+    returns(hadoop.common.RenewDelegationTokenResponseProto);
+    /**
+     Cancels a delegation token signed by OzoneManager.
+    */
+    rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto)
+    returns(hadoop.common.CancelDelegationTokenResponseProto);
+
     /**
      Creates an S3 bucket and creates an ozone mapping for that bucket.
     */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
new file mode 100644
index 0000000..e4a8f2b
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.Signature;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link OzoneSecretManager}.
+ */
+public class TestOzoneSecretManager {
+
+  private OzoneSecretManager<OzoneTokenIdentifier> secretManager;
+  private SecurityConfig securityConfig;
+  private KeyPair keyPair;
+  private long expiryTime;
+  private Text serviceRpcAdd;
+  private OzoneConfiguration conf;
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneSecretManager.class.getSimpleName());
+  private final static Text TEST_USER = new Text("testUser");
+  private long tokenMaxLifetime = 1000 * 20;
+  private long tokenRemoverScanInterval = 1000 * 20;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new OzoneConfiguration();
+    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+    securityConfig = new SecurityConfig(conf);
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+    serviceRpcAdd = new Text("localhost");
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    secretManager.stop();
+    FileUtils.deleteQuietly(new File(BASEDIR));
+  }
+
+  @Test
+  public void testCreateToken() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    OzoneTokenIdentifier identifier =
+        OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
+    // Check basic details.
+    Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
+    Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
+    Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
+
+    validateHash(token.getPassword(), token.getIdentifier());
+  }
+
+  @Test
+  public void testRenewTokenSuccess() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(10 * 5);
+    long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
+    Assert.assertTrue(renewalTime > 0);
+  }
+
+  /**
+   * Tests failure for mismatch in renewer.
+   */
+  @Test
+  public void testRenewTokenFailure() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    LambdaTestUtils.intercept(AccessControlException.class,
+        "rougeUser tries to renew a token", () -> {
+          secretManager.renewToken(token, "rougeUser");
+        });
+  }
+
+  /**
+   * Tests token renew failure due to max time.
+   */
+  @Test
+  public void testRenewTokenFailureMaxTime() throws Exception {
+    secretManager = createSecretManager(conf, 100,
+        100, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(101);
+    LambdaTestUtils.intercept(IOException.class,
+        "testUser tried to renew an expired token", () -> {
+          secretManager.renewToken(token, TEST_USER.toString());
+        });
+  }
+
+  /**
+   * Tests token renew failure due to renewal time.
+   */
+  @Test
+  public void testRenewTokenFailureRenewalTime() throws Exception {
+    secretManager = createSecretManager(conf, 1000 * 10,
+        10, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    Thread.sleep(15);
+    LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
+      secretManager.renewToken(token, TEST_USER.toString());
+    });
+  }
+
+  @Test
+  public void testCreateIdentifier() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    OzoneTokenIdentifier identifier = secretManager.createIdentifier();
+    // Check basic details.
+    Assert.assertTrue(identifier.getOwner().equals(new Text("")));
+    Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
+    Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
+  }
+
+  @Test
+  public void testCancelTokenSuccess() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    secretManager.cancelToken(token, TEST_USER.toString());
+  }
+
+  @Test
+  public void testCancelTokenFailure() throws Exception {
+    secretManager = createSecretManager(conf, tokenMaxLifetime,
+        expiryTime, tokenRemoverScanInterval);
+    secretManager.startThreads(keyPair);
+    Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+        TEST_USER,
+        TEST_USER);
+    LambdaTestUtils.intercept(AccessControlException.class,
+        "rougeUser is not authorized to cancel the token", () -> {
+          secretManager.cancelToken(token, "rougeUser");
+        });
+  }
+
+  /**
+   * Validate hash using public key of KeyPair.
+   */
+  private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+    Signature rsaSignature =
+        Signature.getInstance(securityConfig.getSignatureAlgo(),
+            securityConfig.getProvider());
+    rsaSignature.initVerify(keyPair.getPublic());
+    rsaSignature.update(identifier);
+    Assert.assertTrue(rsaSignature.verify(hash));
+  }
+
+  /**
+   * Create instance of {@link OzoneSecretManager}.
+   */
+  private OzoneSecretManager<OzoneTokenIdentifier> createSecretManager(
+      OzoneConfiguration config, long tokenMaxLife, long expiry, long
+      tokenRemoverScanTime) throws IOException {
+    return new OzoneSecretManager<>(config, tokenMaxLife,
+        expiry, tokenRemoverScanTime, serviceRpcAdd);
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[39/50] [abbrv] hadoop git commit: HDDS-103. SCM CA: Add new security protocol for SCM to expose security related functions. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-103. SCM CA: Add new security protocol for SCM to expose security related functions. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0be8650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0be8650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0be8650

Branch: refs/heads/HDDS-4
Commit: d0be86504421594ed70080ad6bc4faba52c8e437
Parents: 1842ced
Author: Ajay Kumar <aj...@apache.com>
Authored: Sun Oct 28 22:44:41 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/pom.xml                      |   1 +
 .../hdds/protocol/SCMSecurityProtocol.java      |  44 +++++++
 ...MSecurityProtocolClientSideTranslatorPB.java |  99 +++++++++++++++
 .../hdds/protocolPB/SCMSecurityProtocolPB.java  |  35 ++++++
 ...MSecurityProtocolServerSideTranslatorPB.java |  66 ++++++++++
 .../hadoop/hdds/protocolPB/package-info.java    |  22 ++++
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  16 +++
 .../apache/hadoop/ozone/OzoneSecurityUtil.java  |  43 +++++++
 .../src/main/proto/SCMSecurityProtocol.proto    |  66 ++++++++++
 .../apache/hadoop/hdds/scm/HddsServerUtil.java  |  22 ++++
 .../hdds/scm/server/SCMBlockProtocolServer.java |   2 +-
 .../scm/server/SCMSecurityProtocolServer.java   | 121 +++++++++++++++++++
 .../scm/server/StorageContainerManager.java     |  26 +++-
 13 files changed, 558 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 061158c..25774f8 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -239,6 +239,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                   <include>StorageContainerLocationProtocol.proto</include>
                   <include>hdds.proto</include>
                   <include>ScmBlockLocationProtocol.proto</include>
+                  <include>SCMSecurityProtocol.proto</include>
                 </includes>
               </source>
             </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
new file mode 100644
index 0000000..f0ae41c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public interface SCMSecurityProtocol {
+
+  /**
+   * Get SCM signed certificate for DataNode.
+   *
+   * @param dataNodeDetails - DataNode Details.
+   * @param certSignReq             - Certificate signing request.
+   * @return byte[]         - SCM signed certificate.
+   */
+  String getDataNodeCertificate(
+      DatanodeDetailsProto dataNodeDetails,
+      String certSignReq) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..1726282
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * This class is the client-side translator that forwards requests for
+ * {@link SCMSecurityProtocol} to the {@link SCMSecurityProtocolPB} proxy.
+ */
+public class SCMSecurityProtocolClientSideTranslatorPB implements
+    SCMSecurityProtocol, ProtocolTranslator, Closeable {
+
+  /**
+   * RpcController is not used and hence is set to null.
+   */
+  private static final RpcController NULL_RPC_CONTROLLER = null;
+  private final SCMSecurityProtocolPB rpcProxy;
+
+  public SCMSecurityProtocolClientSideTranslatorPB(
+      SCMSecurityProtocolPB rpcProxy) {
+    this.rpcProxy = rpcProxy;
+  }
+
+  /**
+   * Closes this stream and releases any system resources associated
+   * with it. If the stream is already closed then invoking this
+   * method has no effect.
+   *
+   * <p> As noted in {@link AutoCloseable#close()}, cases where the
+   * close may fail require careful attention. It is strongly advised
+   * to relinquish the underlying resources and to internally
+   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+   * the {@code IOException}.
+   *
+   * @throws IOException if an I/O error occurs
+   */
+  @Override
+  public void close() throws IOException {
+    RPC.stopProxy(rpcProxy);
+  }
+
+  /**
+   * Get SCM signed certificate for DataNode.
+   *
+   * @param dataNodeDetails - DataNode Details.
+   * @param certSignReq             - Certificate signing request.
+   * @return byte[]         - SCM signed certificate.
+   */
+  @Override
+  public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
+      String certSignReq) throws IOException {
+    SCMGetDataNodeCertRequestProto.Builder builder =
+        SCMGetDataNodeCertRequestProto
+            .newBuilder()
+            .setCSR(certSignReq)
+            .setDatanodeDetails(dataNodeDetails);
+    try {
+      return rpcProxy
+          .getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
+          .getX509Certificate();
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  /**
+   * Return the proxy object underlying this protocol translator.
+   *
+   * @return the proxy object underlying this protocol translator.
+   */
+  @Override
+  public Object getUnderlyingProxyObject() {
+    return rpcProxy;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
new file mode 100644
index 0000000..cc60c0a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol for security related operations on SCM.
+ */
+
+@ProtocolInfo(protocolName =
+    "org.apache.hadoop.ozone.protocol.SCMSecurityProtocol",
+    protocolVersion = 1)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+public interface SCMSecurityProtocolPB extends
+    SCMSecurityProtocolService.BlockingInterface {
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..40b9652
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto.ResponseCode;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+
+/**
+ * This class is the server-side translator that forwards requests received on
+ * {@link SCMSecurityProtocolPB} to the {@link
+ * SCMSecurityProtocol} server implementation.
+ */
+public class SCMSecurityProtocolServerSideTranslatorPB implements
+    SCMSecurityProtocolPB {
+
+  private final SCMSecurityProtocol impl;
+
+  public SCMSecurityProtocolServerSideTranslatorPB(SCMSecurityProtocol impl) {
+    this.impl = impl;
+  }
+
+  /**
+   * Get SCM signed certificate for DataNode.
+   *
+   * @param controller
+   * @param request
+   * @return SCMGetDataNodeCertResponseProto.
+   */
+  @Override
+  public SCMGetDataNodeCertResponseProto getDataNodeCertificate(
+      RpcController controller, SCMGetDataNodeCertRequestProto request)
+      throws ServiceException {
+    try {
+      String certificate = impl
+          .getDataNodeCertificate(request.getDatanodeDetails(),
+              request.getCSR());
+      SCMGetDataNodeCertResponseProto.Builder builder =
+          SCMGetDataNodeCertResponseProto
+              .newBuilder()
+              .setResponseCode(ResponseCode.success)
+              .setX509Certificate(certificate);
+      return builder.build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
new file mode 100644
index 0000000..4496019
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.protocolPB;
+/**
+ * This package contains classes for wiring HDDS protobuf calls to rpc.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index abaa0f8..f17c7ec 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -159,6 +159,10 @@ public final class ScmConfigKeys {
       "ozone.scm.block.client.port";
   public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
 
+  public static final String OZONE_SCM_SECURITY_SERVICE_PORT_KEY =
+      "ozone.scm.security.service.port";
+  public static final int OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT = 9961;
+
   // Container service client
   public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
       "ozone.scm.client.address";
@@ -175,6 +179,14 @@ public final class ScmConfigKeys {
   public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
       "0.0.0.0";
 
+  // SCM Security service address.
+  public static final String OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY =
+      "ozone.scm.security.service.address";
+  public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY =
+      "ozone.scm.security.service.bind.host";
+  public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT =
+      "0.0.0.0";
+
   public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
       "ozone.scm.datanode.address";
   public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
@@ -217,6 +229,10 @@ public final class ScmConfigKeys {
       "ozone.scm.handler.count.key";
   public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
 
+  public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
+      "ozone.scm.security.handler.count.key";
+  public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
+
   public static final String OZONE_SCM_DEADNODE_INTERVAL =
       "ozone.scm.dead.node.interval";
   public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
new file mode 100644
index 0000000..d51d829
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Ozone security Util class.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneSecurityUtil {
+
+  private OzoneSecurityUtil() {
+  }
+
+  public static boolean isSecurityEnabled(Configuration conf) {
+    return conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+        OZONE_SECURITY_ENABLED_DEFAULT);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
new file mode 100644
index 0000000..81a93c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.hdds.protocol.proto";
+
+option java_outer_classname = "SCMSecurityProtocolProtos";
+
+option java_generic_services = true;
+
+option java_generate_equals_and_hash = true;
+
+package hadoop.hdds;
+
+import "hdds.proto";
+
+/**
+ * Returns a certificate signed by SCM.
+ */
+message SCMGetDataNodeCertResponseProto {
+  enum ResponseCode {
+    success = 1;
+    authenticationFailed = 2;
+    invalidCSR = 3;
+  }
+  required ResponseCode responseCode = 1;
+  required string x509Certificate = 2; // Base64 encoded X509 certificate.
+}
+
+/**
+* This message is send by data node to prove its identity and get an SCM
+* signed certificate.
+*/
+message SCMGetDataNodeCertRequestProto {
+  required DatanodeDetailsProto datanodeDetails = 1;
+  required string CSR = 2;
+}
+
+
+service SCMSecurityProtocolService {
+  /**
+  * Get SCM signed certificate for DataNode.
+  */
+  rpc getDataNodeCertificate (SCMGetDataNodeCertRequestProto) returns (SCMGetDataNodeCertResponseProto);
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 40b3d7a..e355d0a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -161,6 +161,28 @@ public final class HddsServerUtil {
   }
 
   /**
+   * Retrieve the socket address that should be used by scm security server to
+   * service clients.
+   *
+   * @param conf
+   * @return Target InetSocketAddress for the SCM security service.
+   */
+  public static InetSocketAddress getScmSecurityInetAddress(
+      Configuration conf) {
+    final Optional<String> host = getHostNameFromConfigKeys(conf,
+        ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY);
+
+    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
+        ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
+
+    return NetUtils.createSocketAddr(
+        host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+            ":" + port
+            .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+                ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
+  }
+
+  /**
    * Retrieve the socket address that should be used by DataNodes to connect
    * to the SCM.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 3bb284e..2359c30 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -84,7 +84,7 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
 
     RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
         ProtobufRpcEngine.class);
-    // SCM Block Service RPC
+    // SCM Block Service RPC.
     BlockingService blockProtoPbService =
         ScmBlockLocationProtocolProtos.ScmBlockLocationProtocolService
             .newReflectiveBlockingService(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
new file mode 100644
index 0000000..e810c54
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.KerberosInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
+
+  private static final Logger LOGGER = LoggerFactory
+      .getLogger(SCMClientProtocolServer.class);
+  private final OzoneConfiguration config;
+  private final StorageContainerManager scm;
+  private final RPC.Server rpcServer;
+  private final InetSocketAddress rpcAddress;
+
+  SCMSecurityProtocolServer(OzoneConfiguration conf,
+      StorageContainerManager scm) throws IOException {
+    this.config = conf;
+    this.scm = scm;
+
+    final int handlerCount =
+        conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
+            ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
+    rpcAddress = HddsServerUtil
+        .getScmSecurityInetAddress(conf);
+    // SCM security service RPC service.
+    BlockingService secureProtoPbService =
+        SCMSecurityProtocolProtos.SCMSecurityProtocolService
+            .newReflectiveBlockingService(
+                new SCMSecurityProtocolServerSideTranslatorPB(this));
+    this.rpcServer =
+        StorageContainerManager.startRpcServer(
+            conf,
+            rpcAddress,
+            SCMSecurityProtocolPB.class,
+            secureProtoPbService,
+            handlerCount);
+  }
+
+  /**
+   * Get SCM signed certificate for DataNode.
+   *
+   * @param dnDetails   - DataNode Details.
+   * @param certSignReq - Certificate signing request.
+   * @return byte[]         - SCM signed certificate.
+   */
+  @Override
+  public String getDataNodeCertificate(
+      DatanodeDetailsProto dnDetails,
+      String certSignReq) throws IOException {
+    LOGGER.info("Processing CSR for dn {}, UUID: {}", dnDetails.getHostName(),
+        dnDetails.getUuid());
+    // TODO: Call scm to sign the csr.
+    return null;
+  }
+
+  public RPC.Server getRpcServer() {
+    return rpcServer;
+  }
+
+  public InetSocketAddress getRpcAddress() {
+    return rpcAddress;
+  }
+
+  public void start() {
+    LOGGER.info(StorageContainerManager.buildRpcServerStartMessage("Starting"
+        + " RPC server for SCMSecurityProtocolServer.", getRpcAddress()));
+    getRpcServer().start();
+  }
+
+  public void stop() {
+    try {
+      LOGGER.info("Stopping the SCMSecurityProtocolServer.");
+      getRpcServer().stop();
+    } catch (Exception ex) {
+      LOGGER.error("SCMSecurityProtocolServer stop failed.", ex);
+    }
+  }
+
+  public void join() throws InterruptedException {
+    LOGGER.trace("Join RPC server for SCMSecurityProtocolServer.");
+    getRpcServer().join();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2d5b330..b25a402 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
 import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
 import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.io.IOUtils;
@@ -106,8 +107,6 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
@@ -155,6 +154,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
   private final SCMDatanodeProtocolServer datanodeProtocolServer;
   private final SCMBlockProtocolServer blockProtocolServer;
   private final SCMClientProtocolServer clientProtocolServer;
+  private final SCMSecurityProtocolServer securityProtocolServer;
 
   /*
    * State Managers of SCM.
@@ -204,8 +204,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     StorageContainerManager.initMetrics();
     initContainerReportCache(conf);
     // Authenticate SCM if security is enabled
-    if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
-        OZONE_SECURITY_ENABLED_DEFAULT)) {
+    if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
       loginAsSCMUser(conf);
     }
 
@@ -288,6 +287,11 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
         eventQueue);
     blockProtocolServer = new SCMBlockProtocolServer(conf, this);
     clientProtocolServer = new SCMClientProtocolServer(conf, this);
+    if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
+      securityProtocolServer = new SCMSecurityProtocolServer(conf, this);
+    } else {
+      securityProtocolServer = null;
+    }
     httpServer = new StorageContainerManagerHttpServer(conf);
 
     eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
@@ -619,6 +623,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     return clientProtocolServer;
   }
 
+  public SCMSecurityProtocolServer getSecurityProtocolServer() {
+    return securityProtocolServer;
+  }
+
   /**
    * Initialize container reports cache that sent from datanodes.
    *
@@ -721,6 +729,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     LOG.info(buildRpcServerStartMessage("ScmDatanodeProtocl RPC " +
         "server", getDatanodeProtocolServer().getDatanodeRpcAddress()));
     getDatanodeProtocolServer().start();
+    if(getSecurityProtocolServer() != null) {
+      getSecurityProtocolServer().start();
+    }
 
     httpServer.start();
     scmBlockManager.start();
@@ -785,6 +796,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
       LOG.error("Storage Container Manager HTTP server stop failed.", ex);
     }
 
+    if (getSecurityProtocolServer() != null) {
+      getSecurityProtocolServer().stop();
+    }
+
     try {
       LOG.info("Stopping Block Manager Service.");
       scmBlockManager.stop();
@@ -821,6 +836,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
       getBlockProtocolServer().join();
       getClientProtocolServer().join();
       getDatanodeProtocolServer().join();
+      if(getSecurityProtocolServer() != null) {
+        getSecurityProtocolServer().join();
+      }
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
       LOG.info("Interrupted during StorageContainerManager join.");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[40/50] [abbrv] hadoop git commit: HDDS-760. Add asf license to TestCertificateSignRequest. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-760. Add asf license to TestCertificateSignRequest. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd470bfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd470bfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd470bfd

Branch: refs/heads/HDDS-4
Commit: cd470bfd27c878783666ebb151941ae4c2eed812
Parents: d0be865
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 30 09:10:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../certificates/TestCertificateSignRequest.java   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd470bfd/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index a9285df..0b9ef31 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.hdds.security.x509.certificates;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[06/50] [abbrv] hadoop git commit: HDDS-642. Add chill mode exit condition for pipeline availability. Contributed by Yiqun Lin.

Posted by xy...@apache.org.
HDDS-642. Add chill mode exit condition for pipeline availability. Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b71cc7f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b71cc7f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b71cc7f3

Branch: refs/heads/HDDS-4
Commit: b71cc7f33edbbf6a98d1efb330f1c748b5dd6e75
Parents: efc4d91
Author: Ajay Kumar <aj...@apache.org>
Authored: Wed Nov 28 17:45:46 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Wed Nov 28 17:47:57 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |   5 +
 .../common/src/main/resources/ozone-default.xml |   9 ++
 .../scm/chillmode/PipelineChillModeRule.java    | 108 +++++++++++++++++++
 .../hdds/scm/chillmode/SCMChillModeManager.java |  19 +++-
 .../scm/server/StorageContainerManager.java     |   5 +-
 .../scm/chillmode/TestSCMChillModeManager.java  |  81 ++++++++++++--
 6 files changed, 213 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2d28a5b..f16503e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -87,6 +87,11 @@ public final class HddsConfigKeys {
       "hdds.scm.chillmode.min.datanode";
   public static final int HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT = 1;
 
+  public static final String HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK =
+      "hdds.scm.chillmode.pipeline-availability.check";
+  public static final boolean
+      HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = false;
+
   // % of containers which should have at least one reported replica
   // before SCM comes out of chill mode.
   public static final String HDDS_SCM_CHILLMODE_THRESHOLD_PCT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 9f3d7e1..aa22b2b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1232,6 +1232,15 @@
   </property>
 
   <property>
+    <name>hdds.scm.chillmode.pipeline-availability.check</name>
+    <value>false</value>
+    <tag>HDDS,SCM,OPERATION</tag>
+    <description>
+      Boolean value to enable pipeline availability check during SCM chill mode.
+    </description>
+  </property>
+
+  <property>
     <name>hdds.container.action.max.limit</name>
     <value>20</value>
     <tag>DATANODE</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
new file mode 100644
index 0000000..f9a6e59
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Class defining Chill mode exit criteria for Pipelines.
+ */
+public class PipelineChillModeRule
+    implements ChillModeExitRule<PipelineReportFromDatanode>,
+    EventHandler<PipelineReportFromDatanode> {
+  /** Pipeline availability.*/
+  private AtomicBoolean isPipelineAvailable = new AtomicBoolean(false);
+
+  private final PipelineManager pipelineManager;
+  private final SCMChillModeManager chillModeManager;
+
+  PipelineChillModeRule(PipelineManager pipelineManager,
+      SCMChillModeManager manager) {
+    this.pipelineManager = pipelineManager;
+    this.chillModeManager = manager;
+  }
+
+  @Override
+  public boolean validate() {
+    return isPipelineAvailable.get();
+  }
+
+  @Override
+  public void process(PipelineReportFromDatanode report) {
+    // No need to deal with
+  }
+
+  @Override
+  public void cleanup() {
+    // No need to deal with
+  }
+
+  @Override
+  public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
+      EventPublisher publisher) {
+    // If we are already in pipeline available state,
+    // skipping following check.
+    if (validate()) {
+      chillModeManager.validateChillModeExitRules(publisher);
+      return;
+    }
+
+    Pipeline pipeline;
+    Preconditions.checkNotNull(pipelineReportFromDatanode);
+    PipelineReportsProto pipelineReport = pipelineReportFromDatanode
+        .getReport();
+
+    for (PipelineReport report : pipelineReport.getPipelineReportList()) {
+      PipelineID pipelineID = PipelineID
+          .getFromProtobuf(report.getPipelineID());
+      try {
+        pipeline = pipelineManager.getPipeline(pipelineID);
+      } catch (PipelineNotFoundException e) {
+        continue;
+      }
+
+      if (pipeline.getPipelineState() == Pipeline.PipelineState.OPEN) {
+        // ensure there is an OPEN state pipeline and then allowed
+        // to exit chill mode
+        isPipelineAvailable.set(true);
+
+        if (chillModeManager.getInChillMode()) {
+          SCMChillModeManager.getLogger()
+              .info("SCM in chill mode. 1 Pipeline reported, 1 required.");
+        }
+        break;
+      }
+    }
+
+    if (validate()) {
+      chillModeManager.validateChillModeExitRules(publisher);
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
index fa0eaa9..09b998d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
     .NodeRegistrationContainerReport;
 import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -53,14 +54,18 @@ public class SCMChillModeManager implements
       LoggerFactory.getLogger(SCMChillModeManager.class);
   private final boolean isChillModeEnabled;
   private AtomicBoolean inChillMode = new AtomicBoolean(true);
+
   private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
   private Configuration config;
   private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
   private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
+  private static final String PIPELINE_EXIT_RULE = "PipelineChillModeRule";
+
   private final EventQueue eventPublisher;
 
   public SCMChillModeManager(Configuration conf,
-      List<ContainerInfo> allContainers, EventQueue eventQueue) {
+      List<ContainerInfo> allContainers, PipelineManager pipelineManager,
+      EventQueue eventQueue) {
     this.config = conf;
     this.eventPublisher = eventQueue;
     this.isChillModeEnabled = conf.getBoolean(
@@ -70,6 +75,16 @@ public class SCMChillModeManager implements
       exitRules.put(CONT_EXIT_RULE,
           new ContainerChillModeRule(config, allContainers, this));
       exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+
+      if (conf.getBoolean(
+          HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+          HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT)
+          && pipelineManager != null) {
+        PipelineChillModeRule rule = new PipelineChillModeRule(pipelineManager,
+            this);
+        exitRules.put(PIPELINE_EXIT_RULE, rule);
+        eventPublisher.addHandler(SCMEvents.PIPELINE_REPORT, rule);
+      }
       emitChillModeStatus();
     } else {
       exitChillMode(eventQueue);
@@ -84,7 +99,7 @@ public class SCMChillModeManager implements
     eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
   }
 
-  private void validateChillModeExitRules(EventPublisher eventQueue) {
+  public void validateChillModeExitRules(EventPublisher eventQueue) {
     for (ChillModeExitRule exitRule : exitRules.values()) {
       if (!exitRule.validate()) {
         return;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fe00b54..2d27984 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -217,7 +217,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     NodeReportHandler nodeReportHandler =
         new NodeReportHandler(scmNodeManager);
     PipelineReportHandler pipelineReportHandler =
-            new PipelineReportHandler(pipelineManager, conf);
+        new PipelineReportHandler(pipelineManager, conf);
     CommandStatusReportHandler cmdStatusReportHandler =
         new CommandStatusReportHandler();
 
@@ -302,8 +302,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
         (BlockManagerImpl) scmBlockManager);
     scmChillModeManager = new SCMChillModeManager(conf,
-        containerManager.getContainers(),
-        eventQueue);
+        containerManager.getContainers(), pipelineManager, eventQueue);
 
     eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         scmChillModeManager);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
index 0487fb7..ed17796 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -17,15 +17,29 @@
  */
 package org.apache.hadoop.hdds.scm.chillmode;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
 import org.apache.hadoop.hdds.scm.HddsTestUtils;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.BeforeClass;
@@ -33,9 +47,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
 /** Test class for SCMChillModeManager.
  */
 public class TestSCMChillModeManager {
@@ -65,7 +76,7 @@ public class TestSCMChillModeManager {
 
   @Test
   public void testChillModeStateWithNullContainers() {
-    new SCMChillModeManager(config, null, queue);
+    new SCMChillModeManager(config, null, null, queue);
   }
 
   private void testChillMode(int numContainers) throws Exception {
@@ -76,7 +87,8 @@ public class TestSCMChillModeManager {
     for (ContainerInfo container : containers) {
       container.setState(HddsProtos.LifeCycleState.OPEN);
     }
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    scmChillModeManager = new SCMChillModeManager(
+        config, containers, null, queue);
     queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         scmChillModeManager);
     assertTrue(scmChillModeManager.getInChillMode());
@@ -96,7 +108,8 @@ public class TestSCMChillModeManager {
     for (ContainerInfo container : containers) {
       container.setState(HddsProtos.LifeCycleState.CLOSED);
     }
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    scmChillModeManager = new SCMChillModeManager(
+        config, containers, null, queue);
     queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         scmChillModeManager);
     assertTrue(scmChillModeManager.getInChillMode());
@@ -118,7 +131,8 @@ public class TestSCMChillModeManager {
   public void testDisableChillMode() {
     OzoneConfiguration conf = new OzoneConfiguration(config);
     conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
-    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+    scmChillModeManager = new SCMChillModeManager(
+        conf, containers, null, queue);
     assertFalse(scmChillModeManager.getInChillMode());
   }
 
@@ -149,7 +163,8 @@ public class TestSCMChillModeManager {
       container.setState(HddsProtos.LifeCycleState.OPEN);
     }
 
-    scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+    scmChillModeManager = new SCMChillModeManager(
+        config, containers, null, queue);
     queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         scmChillModeManager);
     assertTrue(scmChillModeManager.getInChillMode());
@@ -173,7 +188,8 @@ public class TestSCMChillModeManager {
   private void testChillModeDataNodes(int numOfDns) throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration(config);
     conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
-    scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+    scmChillModeManager = new SCMChillModeManager(
+        conf, containers, null, queue);
     queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
         scmChillModeManager);
     // Assert SCM is in Chill mode.
@@ -212,4 +228,51 @@ public class TestSCMChillModeManager {
     }, 100, 2000 * 9);
   }
 
+  @Test
+  public void testChillModePipelineExitRule() throws Exception {
+    containers = new ArrayList<>();
+    containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+    String storageDir = GenericTestUtils.getTempPath(
+        TestSCMChillModeManager.class.getName() + UUID.randomUUID());
+    try{
+      MockNodeManager nodeManager = new MockNodeManager(true, 1);
+      config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+      // enable pipeline check
+      config.setBoolean(
+          HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK, true);
+
+      PipelineManager pipelineManager = new SCMPipelineManager(config,
+          nodeManager, queue);
+      scmChillModeManager = new SCMChillModeManager(
+          config, containers, pipelineManager, queue);
+      queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+          scmChillModeManager);
+
+      queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+          HddsTestUtils.createNodeRegistrationContainerReport(containers));
+      assertTrue(scmChillModeManager.getInChillMode());
+
+      // simulation a pipeline report to trigger the rule check
+      Pipeline pipeline = pipelineManager.createPipeline(
+          HddsProtos.ReplicationType.STAND_ALONE,
+          HddsProtos.ReplicationFactor.ONE);
+      PipelineReportsProto.Builder reportBuilder = PipelineReportsProto
+          .newBuilder();
+      reportBuilder.addPipelineReport(PipelineReport.newBuilder()
+          .setPipelineID(pipeline.getId().getProtobuf()));
+
+      queue.fireEvent(SCMEvents.PIPELINE_REPORT, new PipelineReportFromDatanode(
+          pipeline.getNodes().get(0), reportBuilder.build()));
+
+      GenericTestUtils.waitFor(() -> {
+        return !scmChillModeManager.getInChillMode();
+      }, 100, 1000 * 10);
+      pipelineManager.close();
+    } finally {
+      config.setBoolean(
+          HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+          false);
+      FileUtil.fullyDelete(new File(storageDir));
+    }
+  }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[04/50] [abbrv] hadoop git commit: YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda)

Posted by xy...@apache.org.
YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda)

Change-Id: Ia6d532c687168448416dfdf46f0ac34bff20e6ca


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7dab8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7dab8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7dab8e

Branch: refs/heads/HDDS-4
Commit: fe7dab8ef55f08cf18c2d62c782c1ab8930a5a15
Parents: 8976439
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:55:16 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 15:03:06 2018 -0800

----------------------------------------------------------------------
 .../runtime/DockerLinuxContainerRuntime.java    | 44 ++++++++------------
 .../runtime/TestDockerContainerRuntime.java     | 15 ++++---
 2 files changed, 24 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 15ff0ff..225bc19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -456,32 +456,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   @Override
   public void prepareContainer(ContainerRuntimeContext ctx)
       throws ContainerExecutionException {
-    Container container = ctx.getContainer();
-
-    // Create volumes when needed.
-    if (nmContext != null
-        && nmContext.getResourcePluginManager().getNameToPlugins() != null) {
-      for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
-          .getNameToPlugins().values()) {
-        DockerCommandPlugin dockerCommandPlugin =
-            plugin.getDockerCommandPluginInstance();
-        if (dockerCommandPlugin != null) {
-          DockerVolumeCommand dockerVolumeCommand =
-              dockerCommandPlugin.getCreateDockerVolumeCommand(
-                  ctx.getContainer());
-          if (dockerVolumeCommand != null) {
-            runDockerVolumeCommand(dockerVolumeCommand, container);
-
-            // After volume created, run inspect to make sure volume properly
-            // created.
-            if (dockerVolumeCommand.getSubCommand().equals(
-                DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
-              checkDockerVolumeCreated(dockerVolumeCommand, container);
-            }
-          }
-        }
-      }
-    }
   }
 
   private void checkDockerVolumeCreated(
@@ -1034,14 +1008,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
       }
     }
 
-    // use plugins to update docker run command.
+    // use plugins to create volume and update docker run command.
     if (nmContext != null
         && nmContext.getResourcePluginManager().getNameToPlugins() != null) {
       for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
           .getNameToPlugins().values()) {
         DockerCommandPlugin dockerCommandPlugin =
             plugin.getDockerCommandPluginInstance();
+
         if (dockerCommandPlugin != null) {
+          // Create volumes when needed.
+          DockerVolumeCommand dockerVolumeCommand =
+              dockerCommandPlugin.getCreateDockerVolumeCommand(
+                  ctx.getContainer());
+          if (dockerVolumeCommand != null) {
+            runDockerVolumeCommand(dockerVolumeCommand, container);
+
+            // After volume created, run inspect to make sure volume properly
+            // created.
+            if (dockerVolumeCommand.getSubCommand().equals(
+                DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
+              checkDockerVolumeCreated(dockerVolumeCommand, container);
+            }
+          }
+          // Update cmd
           dockerCommandPlugin.updateDockerRunCommand(runCommand, container);
         }
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 40ac618..0b5ce3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -1958,16 +1958,16 @@ public class TestDockerContainerRuntime {
     ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
         PrivilegedOperation.class);
 
-    //single invocation expected
+    //Three invocations expected (volume creation, volume check, run container)
     //due to type erasure + mocking, this verification requires a suppress
     // warning annotation on the entire method
-    verify(mockExecutor, times(2))
+    verify(mockExecutor, times(3))
         .executePrivilegedOperation(anyList(), opCaptor.capture(), any(
             File.class), anyMap(), anyBoolean(), anyBoolean());
 
     //verification completed. we need to isolate specific invications.
     // hence, reset mock here
-    Mockito.reset(mockExecutor);
+    //Mockito.reset(mockExecutor);
 
     List<PrivilegedOperation> allCaptures = opCaptor.getAllValues();
 
@@ -2070,10 +2070,8 @@ public class TestDockerContainerRuntime {
 
     try {
       runtime.prepareContainer(containerRuntimeContext);
-
-      checkVolumeCreateCommand();
-
       runtime.launchContainer(containerRuntimeContext);
+      checkVolumeCreateCommand();
     } catch (ContainerExecutionException e) {
       if (expectFail) {
         // Expected
@@ -2166,10 +2164,11 @@ public class TestDockerContainerRuntime {
     ContainerRuntimeContext containerRuntimeContext = builder.build();
 
     runtime.prepareContainer(containerRuntimeContext);
-    checkVolumeCreateCommand();
 
     runtime.launchContainer(containerRuntimeContext);
-    List<String> dockerCommands = readDockerCommands();
+    checkVolumeCreateCommand();
+
+    List<String> dockerCommands = readDockerCommands(3);
 
     int expected = 14;
     int counter = 0;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[05/50] [abbrv] hadoop git commit: YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.

Posted by xy...@apache.org.
YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efc4d91c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efc4d91c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efc4d91c

Branch: refs/heads/HDDS-4
Commit: efc4d91cbeab8a13f6d61cb0e56443adb2d77559
Parents: fe7dab8
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 29 09:34:14 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 29 09:34:14 2018 +0800

----------------------------------------------------------------------
 .../hadoop/yarn/service/webapp/ApiServer.java   | 209 +++++++++++--------
 .../hadoop/yarn/service/ServiceClientTest.java  |   2 +-
 .../yarn/service/client/ServiceClient.java      |   1 +
 3 files changed, 126 insertions(+), 86 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index db831ba..88aeefd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -118,10 +118,13 @@ public class ApiServer {
           @Override
           public Void run() throws YarnException, IOException {
             ServiceClient sc = getServiceClient();
-            sc.init(YARN_CONFIG);
-            sc.start();
-            sc.actionBuild(service);
-            sc.close();
+            try {
+              sc.init(YARN_CONFIG);
+              sc.start();
+              sc.actionBuild(service);
+            } finally {
+              sc.close();
+            }
             return null;
           }
         });
@@ -133,11 +136,14 @@ public class ApiServer {
               @Override
               public ApplicationId run() throws IOException, YarnException {
                 ServiceClient sc = getServiceClient();
-                sc.init(YARN_CONFIG);
-                sc.start();
-                ApplicationId applicationId = sc.actionCreate(service);
-                sc.close();
-                return applicationId;
+                try {
+                  sc.init(YARN_CONFIG);
+                  sc.start();
+                  ApplicationId applicationId = sc.actionCreate(service);
+                  return applicationId;
+                } finally {
+                  sc.close();
+                }
               }
             });
         serviceStatus.setDiagnostics("Application ID: " + applicationId);
@@ -245,29 +251,32 @@ public class ApiServer {
       public Integer run() throws Exception {
         int result = 0;
         ServiceClient sc = getServiceClient();
-        sc.init(YARN_CONFIG);
-        sc.start();
-        Exception stopException = null;
         try {
-          result = sc.actionStop(appName, destroy);
-          if (result == EXIT_SUCCESS) {
-            LOG.info("Successfully stopped service {}", appName);
-          }
-        } catch (Exception e) {
-          LOG.info("Got exception stopping service", e);
-          stopException = e;
-        }
-        if (destroy) {
-          result = sc.actionDestroy(appName);
-          if (result == EXIT_SUCCESS) {
-            LOG.info("Successfully deleted service {}", appName);
+          sc.init(YARN_CONFIG);
+          sc.start();
+          Exception stopException = null;
+          try {
+            result = sc.actionStop(appName, destroy);
+            if (result == EXIT_SUCCESS) {
+              LOG.info("Successfully stopped service {}", appName);
+            }
+          } catch (Exception e) {
+            LOG.info("Got exception stopping service", e);
+            stopException = e;
           }
-        } else {
-          if (stopException != null) {
-            throw stopException;
+          if (destroy) {
+            result = sc.actionDestroy(appName);
+            if (result == EXIT_SUCCESS) {
+              LOG.info("Successfully deleted service {}", appName);
+            }
+          } else {
+            if (stopException != null) {
+              throw stopException;
+            }
           }
+        } finally {
+          sc.close();
         }
-        sc.close();
         return result;
       }
     });
@@ -378,13 +387,16 @@ public class ApiServer {
             @Override
             public Map<String, Long> run() throws YarnException, IOException {
               ServiceClient sc = new ServiceClient();
-              sc.init(YARN_CONFIG);
-              sc.start();
-              Map<String, Long> original = sc.flexByRestService(appName,
-                  Collections.singletonMap(componentName,
-                      component.getNumberOfContainers()));
-              sc.close();
-              return original;
+              try {
+                sc.init(YARN_CONFIG);
+                sc.start();
+                Map<String, Long> original = sc.flexByRestService(appName,
+                    Collections.singletonMap(componentName,
+                        component.getNumberOfContainers()));
+                return original;
+              } finally {
+                sc.close();
+              }
             }
           });
       ServiceStatus status = new ServiceStatus();
@@ -632,12 +644,15 @@ public class ApiServer {
       public Integer run() throws YarnException, IOException {
         int result = 0;
         ServiceClient sc = new ServiceClient();
-        sc.init(YARN_CONFIG);
-        sc.start();
-        result = sc
-            .actionFlex(appName, componentCountStrings);
-        sc.close();
-        return Integer.valueOf(result);
+        try {
+          sc.init(YARN_CONFIG);
+          sc.start();
+          result = sc
+              .actionFlex(appName, componentCountStrings);
+          return Integer.valueOf(result);
+        } finally {
+          sc.close();
+        }
       }
     });
     if (result == EXIT_SUCCESS) {
@@ -658,12 +673,15 @@ public class ApiServer {
       @Override
       public String run() throws YarnException, IOException {
         ServiceClient sc = getServiceClient();
-        sc.init(YARN_CONFIG);
-        sc.start();
-        String newLifeTime = sc.updateLifetime(appName,
-            updateAppData.getLifetime());
-        sc.close();
-        return newLifeTime;
+        try {
+          sc.init(YARN_CONFIG);
+          sc.start();
+          String newLifeTime = sc.updateLifetime(appName,
+              updateAppData.getLifetime());
+          return newLifeTime;
+        } finally {
+          sc.close();
+        }
       }
     });
     ServiceStatus status = new ServiceStatus();
@@ -681,11 +699,14 @@ public class ApiServer {
           @Override public ApplicationId run()
               throws YarnException, IOException {
             ServiceClient sc = getServiceClient();
-            sc.init(YARN_CONFIG);
-            sc.start();
-            ApplicationId appId = sc.actionStartAndGetId(appName);
-            sc.close();
-            return appId;
+            try {
+              sc.init(YARN_CONFIG);
+              sc.start();
+              ApplicationId appId = sc.actionStartAndGetId(appName);
+              return appId;
+            } finally {
+              sc.close();
+            }
           }
         });
     LOG.info("Successfully started service " + appName);
@@ -702,14 +723,17 @@ public class ApiServer {
     ServiceStatus status = new ServiceStatus();
     ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
       ServiceClient sc = getServiceClient();
-      sc.init(YARN_CONFIG);
-      sc.start();
-      if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
-        sc.actionUpgradeExpress(service);
-      } else {
-        sc.initiateUpgrade(service);
+      try {
+        sc.init(YARN_CONFIG);
+        sc.start();
+        if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
+          sc.actionUpgradeExpress(service);
+        } else {
+          sc.initiateUpgrade(service);
+        }
+      } finally {
+        sc.close();
       }
-      sc.close();
       return null;
     });
     LOG.info("Service {} version {} upgrade initialized", service.getName(),
@@ -724,11 +748,14 @@ public class ApiServer {
       final UserGroupInformation ugi) throws IOException, InterruptedException {
     int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
       ServiceClient sc = getServiceClient();
-      sc.init(YARN_CONFIG);
-      sc.start();
-      int exitCode = sc.actionCancelUpgrade(serviceName);
-      sc.close();
-      return exitCode;
+      try {
+        sc.init(YARN_CONFIG);
+        sc.start();
+        int exitCode = sc.actionCancelUpgrade(serviceName);
+        return exitCode;
+      } finally {
+        sc.close();
+      }
     });
     if (result == EXIT_SUCCESS) {
       ServiceStatus status = new ServiceStatus();
@@ -793,10 +820,13 @@ public class ApiServer {
     return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
       int result1;
       ServiceClient sc = getServiceClient();
-      sc.init(YARN_CONFIG);
-      sc.start();
-      result1 = sc.actionUpgrade(service, containers);
-      sc.close();
+      try {
+        sc.init(YARN_CONFIG);
+        sc.start();
+        result1 = sc.actionUpgrade(service, containers);
+      } finally {
+        sc.close();
+      }
       return result1;
     });
   }
@@ -815,12 +845,15 @@ public class ApiServer {
       public Integer run() throws YarnException, IOException {
         int result = 0;
         ServiceClient sc = new ServiceClient();
-        sc.init(YARN_CONFIG);
-        sc.start();
-        result = sc
-            .actionDecommissionInstances(appName, instances);
-        sc.close();
-        return Integer.valueOf(result);
+        try {
+          sc.init(YARN_CONFIG);
+          sc.start();
+          result = sc
+              .actionDecommissionInstances(appName, instances);
+          return Integer.valueOf(result);
+        } finally {
+          sc.close();
+        }
       }
     });
     if (result == EXIT_SUCCESS) {
@@ -840,11 +873,14 @@ public class ApiServer {
 
     return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
       ServiceClient sc = getServiceClient();
-      sc.init(YARN_CONFIG);
-      sc.start();
-      Service app1 = sc.getStatus(serviceName);
-      sc.close();
-      return app1;
+      try {
+        sc.init(YARN_CONFIG);
+        sc.start();
+        Service app1 = sc.getStatus(serviceName);
+        return app1;
+      } finally {
+        sc.close();
+      }
     });
   }
 
@@ -855,12 +891,15 @@ public class ApiServer {
     return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> {
       ComponentContainers[] result;
       ServiceClient sc = getServiceClient();
-      sc.init(YARN_CONFIG);
-      sc.start();
-      result = sc.getContainers(serviceName, componentNames, version,
-          containerStates);
-      sc.close();
-      return result;
+      try {
+        sc.init(YARN_CONFIG);
+        sc.start();
+        result = sc.getContainers(serviceName, componentNames, version,
+            containerStates);
+        return result;
+      } finally {
+        sc.close();
+      }
     });
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index d022614..89366b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient {
 
   public void forceStop() {
     expectedInstances.clear();
-    super.stop();
+    stop();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 713d890..5cdb8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     if (registryClient != null) {
       registryClient.stop();
     }
+    fs.getFileSystem().close();
     super.serviceStop();
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[26/50] [abbrv] hadoop git commit: HDDS-546. Resolve bouncy castle dependency for hadoop-hdds-common. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-546. Resolve bouncy castle dependency for hadoop-hdds-common. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1c6ff7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1c6ff7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1c6ff7e

Branch: refs/heads/HDDS-4
Commit: d1c6ff7ea3f4f02df0d766860187a391975475d1
Parents: b802cb5
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Sep 25 14:19:14 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/pom.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c6ff7e/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 175500a..15ae307 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -59,6 +59,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
           <groupId>io.dropwizard.metrics</groupId>
           <artifactId>metrics-core</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.bouncycastle</groupId>
+          <artifactId>bcprov-jdk15on</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -112,7 +116,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk15on</artifactId>
-      <version>1.49</version>
+      <version>1.54</version>
     </dependency>
   </dependencies>
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[17/50] [abbrv] hadoop git commit: HDDS-5. Enable OzoneManager kerberos auth. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-5. Enable OzoneManager kerberos auth. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2119be45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2119be45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2119be45

Branch: refs/heads/HDDS-4
Commit: 2119be450b6516fe79278870effd13d542fb3d35
Parents: aa7ca15
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon May 14 09:36:57 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800

----------------------------------------------------------------------
 .../common/src/main/resources/ozone-default.xml |  32 +++-
 .../apache/hadoop/ozone/om/OMConfigKeys.java    |   9 +
 .../ozone/om/protocol/OzoneManagerProtocol.java |   6 +
 .../om/protocolPB/OzoneManagerProtocolPB.java   |   4 +
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |   3 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    | 168 +++++++++++++++----
 .../apache/hadoop/ozone/om/OzoneManager.java    |  69 +++++++-
 .../hadoop/ozone/om/OzoneManagerHttpServer.java |   5 +-
 8 files changed, 246 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index a51443c..d3e352b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1400,7 +1400,23 @@
     <name>ozone.scm.kerberos.principal</name>
     <value></value>
     <tag> OZONE, SECURITY</tag>
-    <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+    <description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
+  </property>
+
+  <property>
+    <name>ozone.om.kerberos.keytab.file</name>
+    <value></value>
+    <tag> OZONE, SECURITY</tag>
+    <description> The keytab file used by KSM daemon to login as its
+      service principal. The principal name is configured with
+      hdds.ksm.kerberos.principal.
+    </description>
+  </property>
+  <property>
+    <name>ozone.om.kerberos.principal</name>
+    <value></value>
+    <tag> OZONE, SECURITY</tag>
+    <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
   </property>
 
   <property>
@@ -1412,4 +1428,18 @@
     <value>/etc/security/keytabs/HTTP.keytab</value>
   </property>
 
+  <property>
+    <name>ozone.om.http.kerberos.principal</name>
+    <value>HTTP/_HOST@EXAMPLE.COM</value>
+    <description>
+      KSM http server kerberos principal.
+    </description>
+  </property>
+  <property>
+    <name>ozone.om.http.kerberos.keytab.file</name>
+    <value>/etc/security/keytabs/HTTP.keytab</value>
+    <description>
+      KSM http server kerberos keytab.
+    </description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 739c75e..0119eb5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -86,4 +86,13 @@ public final class OMConfigKeys {
   public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
       "ozone.om.save.metrics.interval";
   public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
+
+  public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om."
+      + "kerberos.keytab.file";
+  public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+      + ".kerberos.principal";
+  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+      "ozone.om.http.kerberos.keytab.file";
+  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+      = "ozone.om.http.kerberos.principal";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index e4cce65..2a4e864 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.ozone.om.protocol;
 
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -25,14 +26,19 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+
+
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneAclInfo;
 import java.io.IOException;
 import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * Protocol to talk to OM.
  */
+@KerberosInfo(
+    serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
 public interface OzoneManagerProtocol {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index e0879d6..27e8f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -18,9 +18,11 @@
 package org.apache.hadoop.ozone.om.protocolPB;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ipc.ProtocolInfo;
 import org.apache.hadoop.ozone.protocol.proto
     .OzoneManagerProtocolProtos.OzoneManagerService;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * Protocol used to communicate with OM.
@@ -28,6 +30,8 @@ import org.apache.hadoop.ozone.protocol.proto
 @ProtocolInfo(protocolName =
     "org.apache.hadoop.ozone.protocol.OzoneManagerProtocol",
     protocolVersion = 1)
+@KerberosInfo(
+    serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface OzoneManagerProtocolPB
     extends OzoneManagerService.BlockingInterface {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 7b0d73d..44cbecc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -468,7 +468,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
      *
      * @throws IOException
      */
-    private OzoneManager createOM() throws IOException {
+    private OzoneManager createOM()
+        throws IOException, AuthenticationException {
       configureOM();
       OMStorage omStore = new OMStorage(conf);
       initializeOmStorage(omStore);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 9c430ad..5c96067 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Properties;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,15 +36,22 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.server.SCMStorage;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.security.KerberosAuthException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -56,13 +64,23 @@ public final class TestSecureOzoneCluster {
   private Logger LOGGER = LoggerFactory
       .getLogger(TestSecureOzoneCluster.class);
 
+  @Rule
+  public Timeout timeout = new Timeout(80000);
+
   private MiniKdc miniKdc;
   private OzoneConfiguration conf;
   private File workDir;
   private static Properties securityProperties;
   private File scmKeytab;
   private File spnegoKeytab;
+  private File omKeyTab;
   private String curUser;
+  private StorageContainerManager scm;
+  private OzoneManager om;
+
+  private static String clusterId;
+  private static String scmId;
+  private static String omId;
 
   @Before
   public void init() {
@@ -71,6 +89,10 @@ public final class TestSecureOzoneCluster {
       startMiniKdc();
       setSecureConfig(conf);
       createCredentialsInKDC(conf, miniKdc);
+
+      clusterId = UUID.randomUUID().toString();
+      scmId = UUID.randomUUID().toString();
+      omId = UUID.randomUUID().toString();
     } catch (IOException e) {
       LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
     } catch (Exception e) {
@@ -78,12 +100,31 @@ public final class TestSecureOzoneCluster {
     }
   }
 
+  @After
+  public void stop() {
+    try {
+      stopMiniKdc();
+      if (scm != null) {
+        scm.stop();
+      }
+      if (om != null) {
+        om.stop();
+      }
+    } catch (Exception e) {
+      LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+    }
+  }
+
   private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
       throws Exception {
     createPrincipal(scmKeytab,
         conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
     createPrincipal(spnegoKeytab,
-        conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+        conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+        conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+    createPrincipal(omKeyTab,
+        conf.get(OMConfigKeys
+            .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
   }
 
   private void createPrincipal(File keytab, String... principal)
@@ -99,6 +140,10 @@ public final class TestSecureOzoneCluster {
     miniKdc.start();
   }
 
+  private void stopMiniKdc() throws Exception {
+    miniKdc.stop();
+  }
+
   private void setSecureConfig(Configuration conf) throws IOException {
     conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
     String host = KerberosUtil.getLocalHostName();
@@ -114,59 +159,56 @@ public final class TestSecureOzoneCluster {
     conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
         "HTTP_SCM/" + host + "@" + realm);
 
+    conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+        "om/" + host + "@" + realm);
+    conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+        "HTTP_KSM/" + host + "@" + realm);
+
     scmKeytab = new File(workDir, "scm.keytab");
     spnegoKeytab = new File(workDir, "http.keytab");
+    omKeyTab = new File(workDir, "om.keytab");
 
     conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
         scmKeytab.getAbsolutePath());
     conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
         spnegoKeytab.getAbsolutePath());
+    conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+        omKeyTab.getAbsolutePath());
 
   }
 
   @Test
   public void testSecureScmStartupSuccess() throws Exception {
+
+    initSCM();
+    scm = StorageContainerManager.createSCM(null, conf);
+    //Reads the SCM Info from SCM instance
+    ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+    Assert.assertEquals(clusterId, scmInfo.getClusterId());
+    Assert.assertEquals(scmId, scmInfo.getScmId());
+  }
+
+  private void initSCM()
+      throws IOException, AuthenticationException {
     final String path = GenericTestUtils
         .getTempPath(UUID.randomUUID().toString());
     Path scmPath = Paths.get(path, "scm-meta");
     conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
     conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
     SCMStorage scmStore = new SCMStorage(conf);
-    String clusterId = UUID.randomUUID().toString();
-    String scmId = UUID.randomUUID().toString();
     scmStore.setClusterId(clusterId);
     scmStore.setScmId(scmId);
     // writes the version file properties
     scmStore.initialize();
-    StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
-    //Reads the SCM Info from SCM instance
-    ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
-    Assert.assertEquals(clusterId, scmInfo.getClusterId());
-    Assert.assertEquals(scmId, scmInfo.getScmId());
   }
 
   @Test
   public void testSecureScmStartupFailure() throws Exception {
-    final String path = GenericTestUtils
-        .getTempPath(UUID.randomUUID().toString());
-    Path scmPath = Paths.get(path, "scm-meta");
-
-    OzoneConfiguration conf = new OzoneConfiguration();
-    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
-    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
-    conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
-    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
-        "scm@" + miniKdc.getRealm());
+    initSCM();
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
 
-    SCMStorage scmStore = new SCMStorage(conf);
-    String clusterId = UUID.randomUUID().toString();
-    String scmId = UUID.randomUUID().toString();
-    scmStore.setClusterId(clusterId);
-    scmStore.setScmId(scmId);
-    // writes the version file properties
-    scmStore.initialize();
     LambdaTestUtils.intercept(IOException.class,
         "Running in secure mode, but config doesn't have a keytab",
         () -> {
@@ -178,28 +220,82 @@ public final class TestSecureOzoneCluster {
     conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
         "/etc/security/keytabs/scm.keytab");
 
+    testCommonKerberosFailures(
+        () -> StorageContainerManager.createSCM(null, conf));
+
+  }
+
+  private void testCommonKerberosFailures(Callable callable) throws Exception {
     LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
-            + "to login: for principal:",
-        () -> {
-          StorageContainerManager.createSCM(null, conf);
-        });
+        + "to login: for principal:", callable);
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "OAuth2");
 
     LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
             + " attribute value for hadoop.security.authentication of OAuth2",
-        () -> {
-          StorageContainerManager.createSCM(null, conf);
-        });
+        callable);
 
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "KERBEROS_SSL");
     LambdaTestUtils.intercept(AuthenticationException.class,
-        "KERBEROS_SSL authentication method not support.",
-        () -> {
-          StorageContainerManager.createSCM(null, conf);
-        });
+        "KERBEROS_SSL authentication method not",
+        callable);
+  }
 
+  /**
+   * Tests the secure KSM Initialization Failure.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testSecureKsmInitializationFailure() throws Exception {
+    initSCM();
+    // Create a secure SCM instance as om client will connect to it
+    scm = StorageContainerManager.createSCM(null, conf);
+
+    final String path = GenericTestUtils
+        .getTempPath(UUID.randomUUID().toString());
+    OMStorage ksmStore = new OMStorage(conf);
+    ksmStore.setClusterId("testClusterId");
+    ksmStore.setScmId("testScmId");
+    // writes the version file properties
+    ksmStore.initialize();
+    conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+        "non-existent-user@EXAMPLE.com");
+    testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
+  }
+
+  /**
+   * Tests the secure KSM Initialization success.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testSecureKsmInitializationSuccess() throws Exception {
+    initSCM();
+    // Create a secure SCM instance as om client will connect to it
+    scm = StorageContainerManager.createSCM(null, conf);
+    LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+    GenericTestUtils
+        .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
+            org.slf4j.event.Level.INFO);
+
+    final String path = GenericTestUtils
+        .getTempPath(UUID.randomUUID().toString());
+    Path metaDirPath = Paths.get(path, "om-meta");
+
+    OMStorage omStore = new OMStorage(conf);
+    omStore.setClusterId("testClusterId");
+    omStore.setScmId("testScmId");
+    // writes the version file properties
+    omStore.initialize();
+    try {
+      om = OzoneManager.createOm(null, conf);
+    } catch (Exception ex) {
+      // Expects timeout failure from scmClient in KSM but KSM user login via
+      // kerberos should succeed
+      Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+    }
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 71dde34..a05b2fd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
 import org.apache.hadoop.ozone.audit.AuditMessage;
 import org.apache.hadoop.ozone.audit.Auditor;
 import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -69,7 +70,10 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
 import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
@@ -115,6 +119,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys
 import static org.apache.hadoop.ozone.om.OMConfigKeys
     .OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
 import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+    .OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+    .OZONE_OM_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
 /**
@@ -123,7 +131,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
 @InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
 public final class OzoneManager extends ServiceRuntimeInfoImpl
     implements OzoneManagerProtocol, OMMXBean, Auditor {
-  private static final Logger LOG =
+  public static final Logger LOG =
       LoggerFactory.getLogger(OzoneManager.class);
 
   private static final AuditLogger AUDIT =
@@ -161,14 +169,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     Preconditions.checkNotNull(conf);
     configuration = conf;
     omStorage = new OMStorage(conf);
-    scmBlockClient = getScmBlockClient(configuration);
-    scmContainerClient = getScmContainerClient(configuration);
     if (omStorage.getState() != StorageState.INITIALIZED) {
       throw new OMException("OM not initialized.",
           ResultCodes.OM_NOT_INITIALIZED);
     }
 
+    scmContainerClient = getScmContainerClient(configuration);
+
     // verifies that the SCM info in the OM Version file is correct.
+    scmBlockClient = getScmBlockClient(configuration);
+
     ScmInfo scmInfo = scmBlockClient.getScmInfo();
     if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
         .getScmId().equals(omStorage.getScmId()))) {
@@ -249,6 +259,35 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
 
 
   /**
+   * Login KSM service user if security and Kerberos are enabled.
+   *
+   * @param  conf
+   * @throws IOException, AuthenticationException
+   */
+  private static void loginKSMUser(OzoneConfiguration conf)
+      throws IOException, AuthenticationException {
+
+    if (SecurityUtil.getAuthenticationMethod(conf).equals
+        (AuthenticationMethod.KERBEROS)) {
+      LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+              + "Principal: {},keytab: {}", conf.get
+              (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+          conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
+
+      UserGroupInformation.setConfiguration(conf);
+
+      InetSocketAddress socAddr = getOmAddress(conf);
+      SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+          OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+    } else {
+      throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
+          (conf) + " authentication method not supported. KSM user login "
+          + "failed.");
+    }
+    LOG.info("KSM login successful.");
+  }
+
+  /**
    * Create a scm block client, used by putKey() and getKey().
    *
    * @return {@link ScmBlockLocationProtocol}
@@ -365,15 +404,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
    * @param argv Command line arguments
    * @param conf OzoneConfiguration
    * @return OM instance
-   * @throws IOException in case OM instance creation fails.
+   * @throws IOException, AuthenticationException in case OM instance
+   *   creation fails.
    */
   @VisibleForTesting
   public static OzoneManager createOm(
-      String[] argv, OzoneConfiguration conf) throws IOException {
+      String[] argv, OzoneConfiguration conf)
+      throws IOException, AuthenticationException {
     return createOm(argv, conf, false);
   }
 
-
   /**
    * Constructs OM instance based on command line arguments.
    *
@@ -381,10 +421,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
    * @param conf OzoneConfiguration
    * @param printBanner if true then log a verbose startup message.
    * @return OM instance
-   * @throws IOException in case OM instance creation fails.
+   * @throws IOException, AuthenticationException in case OM instance
+   *   creation fails.
    */
   private static OzoneManager createOm(String[] argv,
-      OzoneConfiguration conf, boolean printBanner) throws IOException {
+      OzoneConfiguration conf, boolean printBanner)
+      throws IOException, AuthenticationException {
     if (!isHddsEnabled(conf)) {
       System.err.println("OM cannot be started in secure mode or when " +
           OZONE_ENABLED + " is set to false");
@@ -396,6 +438,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
       terminate(1);
       return null;
     }
+    // Authenticate KSM if security is enabled
+    if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+      loginKSMUser(conf);
+    }
     switch (startOpt) {
     case INIT:
       if (printBanner) {
@@ -569,7 +615,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     keyManager.start(configuration);
 
     httpServer = new OzoneManagerHttpServer(configuration, this);
-    httpServer.start();
+    try {
+      httpServer.start();
+    } catch (Exception ex) {
+      // Allow OM to start as Http Server failure is not fatal.
+      LOG.error("OM HttpServer failed to start.", ex);
+    }
     registerMXBean();
     setStartTime();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index bd6ab69..be12fa8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.ozone.om;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.hdds.server.BaseHttpServer;
 
@@ -65,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
   }
 
   @Override protected String getKeytabFile() {
-    return OMConfigKeys.OZONE_OM_KEYTAB_FILE;
+    return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
   }
 
   @Override protected String getSpnegoPrincipal() {
-    return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+    return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
   }
 
   @Override protected String getEnabledKey() {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[47/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-759

Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-759


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18c9db98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18c9db98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18c9db98

Branch: refs/heads/HDDS-4
Commit: 18c9db980f8d3bcd01e3a9fa0f59861e26ffff89
Parents: 96bd574
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:21:44 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java  | 2 +-
 .../security/x509/certificates/TestCertificateSignRequest.java     | 2 +-
 .../hdds/security/x509/certificates/TestRootCertificate.java       | 2 +-
 .../hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 97627ca..2826e55 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -50,7 +50,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 
 /**
  * A class that deals with all Security related configs in HDDS.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index e8de1ea..9328c50 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -43,7 +43,7 @@ import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 import java.util.UUID;
 
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 
 public class TestCertificateSignRequest {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 16245a6..e581dc8 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -46,7 +46,7 @@ import java.time.Instant;
 import java.util.Date;
 import java.util.UUID;
 
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 
 /**
  * Test Class for Root Certificate generation.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
index f9541a2..08761f4 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdds.security.x509.keys;
 
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[33/50] [abbrv] hadoop git commit: HDDS-704. Fix the Dependency convergence issue on HDDS-4. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-704. Fix the Dependency convergence issue on HDDS-4. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/950a4733
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/950a4733
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/950a4733

Branch: refs/heads/HDDS-4
Commit: 950a4733ac86a0284ec46a1060ea3862f15181d3
Parents: b4c0280
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Oct 19 21:09:51 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/pom.xml | 7 -------
 1 file changed, 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/950a4733/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 30dcc58..061158c 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -112,13 +112,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>commons-pool2</artifactId>
       <version>2.6.0</version>
     </dependency>
-
-    <dependency>
-      <groupId>org.bouncycastle</groupId>
-      <artifactId>bcprov-jdk15on</artifactId>
-      <version>1.54</version>
-    </dependency>
-
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcpkix-jdk15on</artifactId>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[28/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed Certificate. Contributed by Anu Engineer.

Posted by xy...@apache.org.
HDDS-548. Create a Self-Signed Certificate. Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87acc150
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87acc150
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87acc150

Branch: refs/heads/HDDS-4
Commit: 87acc1508a4bafc8cd3d77ffa684bf9871f52698
Parents: d1c6ff7
Author: Ajay Kumar <aj...@apache.com>
Authored: Fri Sep 28 06:52:56 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |  16 ++
 .../hdds/security/x509/HDDSKeyGenerator.java    |  99 -------
 .../hdds/security/x509/HDDSKeyPEMWriter.java    | 254 ------------------
 .../hdds/security/x509/SecurityConfig.java      | 105 +++++---
 .../certificates/SelfSignedCertificate.java     | 212 +++++++++++++++
 .../x509/certificates/package-info.java         |  22 ++
 .../x509/exceptions/CertificateException.java   |  63 +++++
 .../x509/exceptions/SCMSecurityException.java   |  64 +++++
 .../security/x509/exceptions/package-info.java  |  23 ++
 .../security/x509/keys/HDDSKeyGenerator.java    | 106 ++++++++
 .../security/x509/keys/HDDSKeyPEMWriter.java    | 255 ++++++++++++++++++
 .../hdds/security/x509/keys/package-info.java   |  23 ++
 .../security/x509/TestHDDSKeyGenerator.java     |  81 ------
 .../security/x509/TestHDDSKeyPEMWriter.java     | 213 ---------------
 .../x509/certificates/TestRootCertificate.java  | 258 +++++++++++++++++++
 .../x509/certificates/package-info.java         |  22 ++
 .../x509/keys/TestHDDSKeyGenerator.java         |  87 +++++++
 .../x509/keys/TestHDDSKeyPEMWriter.java         | 216 ++++++++++++++++
 .../hdds/security/x509/keys/package-info.java   |  22 ++
 .../hadoop/hdds/security/x509/package-info.java |  22 ++
 20 files changed, 1484 insertions(+), 679 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2a1404a..ef1194b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -131,4 +131,20 @@ public final class HddsConfigKeys {
   public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
       + ".name";
   public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
+
+  /**
+   * Maximum duration of certificates issued by SCM including Self-Signed Roots.
+   * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
+   * Default value is 5 years and written as P1865D.
+   */
+  public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
+  // Limit Certificate duration to a max value of 5 years.
+  public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+
+  public static final String HDDS_X509_SIGNATURE_ALGO =
+      "hdds.x509.signature.algorithm";
+  public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
+
+
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
deleted file mode 100644
index cb411b2..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-
-/** A class to generate Key Pair for use with Certificates. */
-public class HDDSKeyGenerator {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(HDDSKeyGenerator.class);
-  private final SecurityConfig securityConfig;
-
-  /**
-   * Constructor for HDDSKeyGenerator.
-   *
-   * @param configuration - config
-   */
-  public HDDSKeyGenerator(Configuration configuration) {
-    this.securityConfig = new SecurityConfig(configuration);
-  }
-
-  /**
-   * Returns the Security config used for this object.
-   * @return SecurityConfig
-   */
-  public SecurityConfig getSecurityConfig() {
-    return securityConfig;
-  }
-
-  /**
-   * Use Config to generate key.
-   *
-   * @return KeyPair
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   */
-  public KeyPair generateKey() throws NoSuchProviderException,
-      NoSuchAlgorithmException {
-    return generateKey(securityConfig.getSize(),
-        securityConfig.getAlgo(), securityConfig.getProvider());
-  }
-
-  /**
-   * Specify the size -- all other parameters are used from config.
-   *
-   * @param size - int, valid key sizes.
-   * @return KeyPair
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   */
-  public KeyPair generateKey(int size) throws
-      NoSuchProviderException, NoSuchAlgorithmException {
-    return generateKey(size,
-        securityConfig.getAlgo(), securityConfig.getProvider());
-  }
-
-  /**
-   * Custom Key Generation, all values are user provided.
-   *
-   * @param size - Key Size
-   * @param algorithm - Algorithm to use
-   * @param provider - Security provider.
-   * @return KeyPair.
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   */
-  public KeyPair generateKey(int size, String algorithm, String provider)
-      throws NoSuchProviderException, NoSuchAlgorithmException {
-    LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
-        size, algorithm, provider);
-    KeyPairGenerator generator = KeyPairGenerator
-        .getInstance(algorithm, provider);
-    generator.initialize(size);
-    return generator.generateKeyPair();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
deleted file mode 100644
index 6ca7584..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.output.FileWriterWithEncoding;
-import org.apache.hadoop.conf.Configuration;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyPair;
-import java.util.Set;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * We store all Key material in good old PEM files.
- * This helps in avoiding dealing will persistent
- * Java KeyStore issues. Also when debugging,
- * general tools like OpenSSL can be used to read and
- * decode these files.
- */
-public class HDDSKeyPEMWriter {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
-  private final Path location;
-  private final SecurityConfig securityConfig;
-  private Set<PosixFilePermission> permissionSet =
-      Stream.of(OWNER_READ, OWNER_WRITE,  OWNER_EXECUTE)
-          .collect(Collectors.toSet());
-  private Supplier<Boolean> isPosixFileSystem;
-  public final static String PRIVATE_KEY = "PRIVATE KEY";
-  public final static String PUBLIC_KEY = "PUBLIC KEY";
-  public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
-  /*
-    Creates an HDDS Key Writer.
-
-    @param configuration - Configuration
-   */
-  public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
-    Preconditions.checkNotNull(configuration, "Config cannot be null");
-    this.securityConfig = new SecurityConfig(configuration);
-    isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
-    this.location = securityConfig.getKeyLocation();
-  }
-
-  /**
-   * Checks if File System supports posix style security permissions.
-   *
-   * @return True if it supports posix.
-   */
-  private static Boolean isPosix() {
-    return FileSystems.getDefault().supportedFileAttributeViews()
-        .contains("posix");
-  }
-
-  /**
-   * Returns the Permission set.
-   * @return Set
-   */
-  @VisibleForTesting
-  public Set<PosixFilePermission> getPermissionSet() {
-    return permissionSet;
-  }
-
-  /**
-   * Returns the Security config used for this object.
-   * @return SecurityConfig
-   */
-  public SecurityConfig getSecurityConfig() {
-    return securityConfig;
-  }
-
-  /**
-   * This function is used only for testing.
-   *
-   * @param isPosixFileSystem - Sets a boolean function for mimicking
-   * files systems that are not posix.
-   */
-  @VisibleForTesting
-  public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
-    this.isPosixFileSystem = isPosixFileSystem;
-  }
-
-  /**
-   * Writes a given key using the default config options.
-   *
-   * @param keyPair - Key Pair to write to file.
-   * @throws IOException
-   */
-  public void writeKey(KeyPair keyPair) throws IOException {
-    writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
-        securityConfig.getPublicKeyName(), false);
-  }
-
-  /**
-   * Writes a given key using default config options.
-   *
-   * @param keyPair - Key pair to write
-   * @param overwrite - Overwrites the keys if they already exist.
-   * @throws IOException
-   */
-  public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
-    writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
-        securityConfig.getPublicKeyName(), overwrite);
-  }
-
-  /**
-   * Writes a given key using default config options.
-   *
-   * @param basePath - The location to write to, override the config values.
-   * @param keyPair - Key pair to write
-   * @param overwrite - Overwrites the keys if they already exist.
-   * @throws IOException
-   */
-  public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
-      throws IOException {
-    writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
-        securityConfig.getPublicKeyName(), overwrite);
-  }
-
-  /**
-   * Helper function that actually writes data to the files.
-   *
-   * @param basePath - base path to write key
-   * @param keyPair - Key pair to write to file.
-   * @param privateKeyFileName - private key file name.
-   * @param publicKeyFileName - public key file name.
-   * @param force - forces overwriting the keys.
-   * @throws IOException
-   */
-  private synchronized void writeKey(Path basePath, KeyPair keyPair,
-      String privateKeyFileName, String publicKeyFileName, boolean force)
-      throws IOException {
-    checkPreconditions(basePath);
-
-    File privateKeyFile =
-        Paths.get(location.toString(), privateKeyFileName).toFile();
-    File publicKeyFile =
-        Paths.get(location.toString(), publicKeyFileName).toFile();
-    checkKeyFile(privateKeyFile, force, publicKeyFile);
-
-    try (PemWriter privateKeyWriter = new PemWriter(new
-        FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
-      privateKeyWriter.writeObject(
-          new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
-    }
-
-    try (PemWriter publicKeyWriter = new PemWriter(new
-        FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
-      publicKeyWriter.writeObject(
-          new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
-    }
-    Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
-    Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
-  }
-
-  /**
-   * Checks if private and public key file already exists. Throws IOException
-   * if file exists and force flag is set to false, else will delete the
-   * existing file.
-   *
-   * @param privateKeyFile - Private key file.
-   * @param force - forces overwriting the keys.
-   * @param publicKeyFile - public key file.
-   * @throws IOException
-   */
-  private void checkKeyFile(File privateKeyFile, boolean force,
-      File publicKeyFile) throws IOException {
-    if (privateKeyFile.exists() && force) {
-      if (!privateKeyFile.delete()) {
-        throw new IOException("Unable to delete private key file.");
-      }
-    }
-
-    if (publicKeyFile.exists() && force) {
-      if (!publicKeyFile.delete()) {
-        throw new IOException("Unable to delete public key file.");
-      }
-    }
-
-    if (privateKeyFile.exists()) {
-      throw new IOException("Private Key file already exists.");
-    }
-
-    if (publicKeyFile.exists()) {
-      throw new IOException("Public Key file already exists.");
-    }
-  }
-
-  /**
-   * Checks if base path exists and sets file permissions.
-   *
-   * @param basePath - base path to write key
-   * @throws IOException
-   */
-  private void checkPreconditions(Path basePath) throws IOException {
-    Preconditions.checkNotNull(basePath, "Base path cannot be null");
-    if (!isPosixFileSystem.get()) {
-      LOG.error("Keys cannot be stored securely without POSIX file system "
-          + "support for now.");
-      throw new IOException("Unsupported File System for pem file.");
-    }
-
-    if (Files.exists(basePath)) {
-      // Not the end of the world if we reset the permissions on an existing
-      // directory.
-      Files.setPosixFilePermissions(basePath, permissionSet);
-    } else {
-      boolean success = basePath.toFile().mkdirs();
-      if (!success) {
-        LOG.error("Unable to create the directory for the "
-            + "location. Location: {}", basePath);
-        throw new IOException("Unable to create the directory for the "
-            + "location. Location:" + basePath);
-      }
-      Files.setPosixFilePermissions(basePath, permissionSet);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 896a379..9231c8a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -29,9 +29,10 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.Provider;
 import java.security.Security;
+import java.time.Duration;
 
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
@@ -43,11 +44,17 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
 
 /**
- * A class that deals with all Security related configs in HDDDS.
- * It is easier to have all Java code related to config in a single place.
+ * A class that deals with all Security related configs in HDDS.
+ *
+ * This class allows security configs to be read and used consistently across
+ * all of security related code base.
  */
 public class SecurityConfig {
   private static final Logger LOG =
@@ -55,15 +62,17 @@ public class SecurityConfig {
   private static volatile Provider provider;
   private final Configuration configuration;
   private final int size;
-  private final String algo;
+  private final String keyAlgo;
   private final String providerString;
   private final String metadatDir;
   private final String keyDir;
-  private final String privateKeyName;
-  private final String publicKeyName;
+  private final String privateKeyFileName;
+  private final String publicKeyFileName;
+  private final Duration certDuration;
+  private final String x509SignatureAlgo;
 
   /**
-   * Constructs a HDDSKeyGenerator.
+   * Constructs a SecurityConfig.
    *
    * @param configuration - HDDS Configuration
    */
@@ -71,10 +80,10 @@ public class SecurityConfig {
     Preconditions.checkNotNull(configuration, "Configuration cannot be null");
     this.configuration = configuration;
     this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
-    this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+    this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
         HDDS_DEFAULT_KEY_ALGORITHM);
     this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
-            HDDS_DEFAULT_SECURITY_PROVIDER);
+        HDDS_DEFAULT_SECURITY_PROVIDER);
 
     // Please Note: To make it easy for our customers we will attempt to read
     // HDDS metadata dir and if that is not set, we will use Ozone directory.
@@ -86,11 +95,17 @@ public class SecurityConfig {
         + " null. Please check configs.");
     this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
         HDDS_KEY_DIR_NAME_DEFAULT);
-    this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+    this.privateKeyFileName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
         HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
-    this.publicKeyName =  this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+    this.publicKeyFileName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
         HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
 
+    String durationString = this.configuration.get(HDDS_X509_MAX_DURATION,
+        HDDS_X509_MAX_DURATION_DEFAULT);
+    this.certDuration = Duration.parse(durationString);
+    this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
+        HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+
     // First Startup -- if the provider is null, check for the provider.
     if (SecurityConfig.provider == null) {
       synchronized (SecurityConfig.class) {
@@ -105,39 +120,38 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the Provider name.
-   * @return String Provider name.
-   */
-  public String getProviderString() {
-    return providerString;
-  }
-
-  /**
-   * Returns the public key file name.
+   * Returns the public key file name, This is used for storing the public
+   * keys on disk.
+   *
    * @return String, File name used for public keys.
    */
-  public String getPublicKeyName() {
-    return publicKeyName;
+  public String getPublicKeyFileName() {
+    return publicKeyFileName;
   }
 
   /**
-   * Returns the private key file name.
+   * Returns the private key file name.This is used for storing the private
+   * keys on disk.
+   *
    * @return String, File name used for private keys.
    */
-  public String getPrivateKeyName() {
-    return privateKeyName;
+  public String getPrivateKeyFileName() {
+    return privateKeyFileName;
   }
 
   /**
    * Returns the File path to where keys are stored.
-   * @return  String Key location.
+   *
+   * @return String Key location.
    */
   public Path getKeyLocation() {
     return Paths.get(metadatDir, keyDir);
   }
 
   /**
-   * Gets the Key Size.
+   * Gets the Key Size, The default key size is 2048, since the default
+   * algorithm used is RSA. User can change this by setting the "hdds.key
+   * .len" in configuration.
    *
    * @return key size.
    */
@@ -146,7 +160,8 @@ public class SecurityConfig {
   }
 
   /**
-   * Gets provider.
+   * Returns the Provider name. SCM defaults to using Bouncy Castle and will
+   * return "BC".
    *
    * @return String Provider name.
    */
@@ -155,22 +170,48 @@ public class SecurityConfig {
   }
 
   /**
-   * Returns the Key generation Algorithm used.
+   * Returns the Key generation Algorithm used.  User can change this by
+   * setting the "hdds.key.algo" in configuration.
    *
    * @return String Algo.
    */
-  public String getAlgo() {
-    return algo;
+  public String getKeyAlgo() {
+    return keyAlgo;
+  }
+
+  /**
+   * Returns the X.509 Signature Algorithm used. This can be changed by setting
+   * "hdds.x509.signature.algorithm" to the new name. The default algorithm
+   * is SHA256withRSA.
+   *
+   * @return String
+   */
+  public String getSignatureAlgo() {
+    return x509SignatureAlgo;
   }
 
   /**
    * Returns the Configuration used for initializing this SecurityConfig.
-   * @return  Configuration
+   *
+   * @return Configuration
    */
   public Configuration getConfiguration() {
     return configuration;
   }
 
+  /**
+   * Returns the maximum length a certificate can be valid in SCM. The
+   * default value is 5 years. This can be changed by setting
+   * "hdds.x509.max.duration" in configuration. The formats accepted are
+   * based on the ISO-8601 duration format PnDTnHnMn.nS
+   *
+   * Default value is 5 years and written as P1865D.
+   *
+   * @return Duration.
+   */
+  public Duration getMaxCertificateDuration() {
+    return this.certDuration;
+  }
 
   /**
    * Adds a security provider dynamically if it is not loaded already.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
new file mode 100644
index 0000000..fef7ac3
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.util.Time;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.util.Date;
+
+/**
+ * A Self Signed Certificate with CA basic constraint can be used to boot-strap
+ * a certificate infra-structure, if no external certificate is provided.
+ */
+public final class SelfSignedCertificate {
+  private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+  private String subject;
+  private String clusterID;
+  private String scmID;
+  private Date beginDate;
+  private Date endDate;
+  private KeyPair key;
+  private SecurityConfig config;
+  private boolean isCA;
+
+  /**
+   * Private Ctor invoked only via Builder Interface.
+   * @param subject - Subject
+   * @param scmID - SCM ID
+   * @param clusterID - Cluster ID
+   * @param beginDate - NotBefore
+   * @param endDate - Not After
+   * @param configuration - SCM Config
+   * @param keyPair - KeyPair
+   * @param ca - isCA?
+   */
+  private SelfSignedCertificate(String subject, String scmID, String clusterID,
+      Date beginDate, Date endDate, SecurityConfig configuration,
+      KeyPair keyPair, boolean ca) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+    this.beginDate = beginDate;
+    this.endDate = endDate;
+    config = configuration;
+    this.key = keyPair;
+    this.isCA = ca;
+  }
+
+  @VisibleForTesting
+  public static String getNameFormat() {
+    return NAME_FORMAT;
+  }
+
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  private X509CertificateHolder generateCertificate()
+      throws OperatorCreationException, CertIOException {
+    // For the Root Certificate we form the name from Subject, SCM ID and
+    // Cluster ID.
+    String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
+    X500Name name = new X500Name(dnName);
+    byte[] encoded = key.getPublic().getEncoded();
+    SubjectPublicKeyInfo publicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(encoded);
+
+
+    ContentSigner contentSigner =
+        new JcaContentSignerBuilder(
+            config.getSignatureAlgo()).build(key.getPrivate());
+
+    // Please note: Since this is a root certificate we use "ONE" as the
+    // serial number. Also note that skip enforcing locale or UTC. We are
+    // trying to operate at the Days level, hence Time zone is also skipped for
+    // now.
+    BigInteger serial = BigInteger.ONE;
+    if (!isCA) {
+      serial = new BigInteger(Long.toString(Time.monotonicNow()));
+    }
+
+    X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
+        serial, beginDate, endDate, name, publicKeyInfo);
+
+    if (isCA) {
+      builder.addExtension(Extension.basicConstraints, true,
+          new BasicConstraints(true));
+    }
+    return builder.build(contentSigner);
+  }
+
+  /**
+   * Builder class for Root Certificates.
+   */
+  public static class Builder {
+    private String subject;
+    private String clusterID;
+    private String scmID;
+    private Date beginDate;
+    private Date endDate;
+    private KeyPair key;
+    private SecurityConfig config;
+    private boolean isCA;
+
+    public Builder setConfiguration(Configuration configuration) {
+      this.config = new SecurityConfig(configuration);
+      return this;
+    }
+
+    public Builder setKey(KeyPair keyPair) {
+      this.key = keyPair;
+      return this;
+    }
+
+    public Builder setSubject(String subjectString) {
+      this.subject = subjectString;
+      return this;
+    }
+
+    public Builder setClusterID(String s) {
+      this.clusterID = s;
+      return this;
+    }
+
+    public Builder setScmID(String s) {
+      this.scmID = s;
+      return this;
+    }
+
+    public Builder setBeginDate(Date date) {
+      this.beginDate = new Date(date.toInstant().toEpochMilli());
+      return this;
+    }
+
+    public Builder setEndDate(Date date) {
+      this.endDate = new Date(date.toInstant().toEpochMilli());
+      return this;
+    }
+
+    public Builder makeCA() {
+      isCA = true;
+      return this;
+    }
+
+    public X509CertificateHolder build() throws SCMSecurityException {
+      Preconditions.checkNotNull(key, "Key cannot be null");
+      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+          "be blank");
+
+      Preconditions.checkArgument(beginDate.before(endDate), "Certificate " +
+          "begin date should be before end date");
+
+      Duration certDuration = Duration.between(beginDate.toInstant(),
+          endDate.toInstant());
+      Preconditions.checkArgument(
+          certDuration.compareTo(config.getMaxCertificateDuration()) < 0,
+          "Certificate life time cannot be greater than max configured value.");
+
+
+      SelfSignedCertificate rootCertificate =
+          new SelfSignedCertificate(this.subject,
+          this.scmID, this.clusterID, this.beginDate, this.endDate,
+          this.config, key, isCA);
+      try {
+        return rootCertificate.generateCertificate();
+      } catch (OperatorCreationException | CertIOException e) {
+        throw new CertificateException("Unable to create root certificate.",
+            e.getCause());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..e88737c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Utils for Certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
new file mode 100644
index 0000000..4d2dcfd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Certificate Exceptions from the SCM Security layer.
+ */
+public class CertificateException extends SCMSecurityException {
+
+  /**
+   * Ctor.
+   * @param message - Error Message.
+   */
+  public CertificateException(String message) {
+    super(message);
+  }
+
+  /**
+   * Ctor.
+   * @param message - Message.
+   * @param cause  - Actual cause.
+   */
+  public CertificateException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  /**
+   * Ctor.
+   * @param cause - Base Exception.
+   */
+  public CertificateException(Throwable cause) {
+    super(cause);
+  }
+
+  /**
+   * Ctor.
+   * @param message - Error Message
+   * @param cause  - Cause
+   * @param enableSuppression - Enable suppression.
+   * @param writableStackTrace - Writable stack trace.
+   */
+  public CertificateException(String message, Throwable cause,
+      boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
new file mode 100644
index 0000000..e1f6296
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends Exception {
+
+  /**
+   * Ctor.
+   * @param message - Error Message.
+   */
+  public SCMSecurityException(String message) {
+    super(message);
+  }
+
+  /**
+   * Ctor.
+   * @param message - Message.
+   * @param cause  - Actual cause.
+   */
+  public SCMSecurityException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  /**
+   * Ctor.
+   * @param cause - Base Exception.
+   */
+  public SCMSecurityException(Throwable cause) {
+    super(cause);
+  }
+
+
+  /**
+   * Ctor.
+   * @param message - Error Message
+   * @param cause  - Cause
+   * @param enableSuppression - Enable suppression.
+   * @param writableStackTrace - Writable stack trace.
+   */
+  public SCMSecurityException(String message, Throwable cause,
+      boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
new file mode 100644
index 0000000..afcc474
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by X.509 security classes.
+ */
+package org.apache.hadoop.hdds.security.x509.exceptions;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
new file mode 100644
index 0000000..99873cb
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(HDDSKeyGenerator.class);
+  private final SecurityConfig securityConfig;
+
+  /**
+   * Constructor for HDDSKeyGenerator.
+   *
+   * @param configuration - config
+   */
+  public HDDSKeyGenerator(Configuration configuration) {
+    this.securityConfig = new SecurityConfig(configuration);
+  }
+
+  /**
+   * Returns the Security config used for this object.
+   * @return SecurityConfig
+   */
+  public SecurityConfig getSecurityConfig() {
+    return securityConfig;
+  }
+
+  /**
+   * Use Config to generate key.
+   *
+   * @return KeyPair
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   */
+  public KeyPair generateKey() throws NoSuchProviderException,
+      NoSuchAlgorithmException {
+    return generateKey(securityConfig.getSize(),
+        securityConfig.getKeyAlgo(), securityConfig.getProvider());
+  }
+
+  /**
+   * Specify the size -- all other parameters are used from config.
+   *
+   * @param size - int, valid key sizes.
+   * @return KeyPair
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   */
+  public KeyPair generateKey(int size) throws
+      NoSuchProviderException, NoSuchAlgorithmException {
+    return generateKey(size,
+        securityConfig.getKeyAlgo(), securityConfig.getProvider());
+  }
+
+  /**
+   * Custom Key Generation, all values are user provided.
+   *
+   * @param size - Key Size
+   * @param algorithm - Algorithm to use
+   * @param provider - Security provider.
+   * @return KeyPair.
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   */
+  public KeyPair generateKey(int size, String algorithm, String provider)
+      throws NoSuchProviderException, NoSuchAlgorithmException {
+    LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+        size, algorithm, provider);
+    KeyPairGenerator generator = KeyPairGenerator
+        .getInstance(algorithm, provider);
+    generator.initialize(size);
+    return generator.generateKeyPair();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..95be1c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+  private final Path location;
+  private final SecurityConfig securityConfig;
+  private Set<PosixFilePermission> permissionSet =
+      Stream.of(OWNER_READ, OWNER_WRITE,  OWNER_EXECUTE)
+          .collect(Collectors.toSet());
+  private Supplier<Boolean> isPosixFileSystem;
+  public final static String PRIVATE_KEY = "PRIVATE KEY";
+  public final static String PUBLIC_KEY = "PUBLIC KEY";
+  public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+  /*
+    Creates an HDDS Key Writer.
+
+    @param configuration - Configuration
+   */
+  public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+    Preconditions.checkNotNull(configuration, "Config cannot be null");
+    this.securityConfig = new SecurityConfig(configuration);
+    isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+    this.location = securityConfig.getKeyLocation();
+  }
+
+  /**
+   * Checks if File System supports posix style security permissions.
+   *
+   * @return True if it supports posix.
+   */
+  private static Boolean isPosix() {
+    return FileSystems.getDefault().supportedFileAttributeViews()
+        .contains("posix");
+  }
+
+  /**
+   * Returns the Permission set.
+   * @return Set
+   */
+  @VisibleForTesting
+  public Set<PosixFilePermission> getPermissionSet() {
+    return permissionSet;
+  }
+
+  /**
+   * Returns the Security config used for this object.
+   * @return SecurityConfig
+   */
+  public SecurityConfig getSecurityConfig() {
+    return securityConfig;
+  }
+
+  /**
+   * This function is used only for testing.
+   *
+   * @param isPosixFileSystem - Sets a boolean function for mimicking
+   * files systems that are not posix.
+   */
+  @VisibleForTesting
+  public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+    this.isPosixFileSystem = isPosixFileSystem;
+  }
+
+  /**
+   * Writes a given key using the default config options.
+   *
+   * @param keyPair - Key Pair to write to file.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(KeyPair keyPair) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), false);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), overwrite);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param basePath - The location to write to, override the config values.
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException - On I/O failure.
+   */
+  public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+      throws IOException {
+    writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
+        securityConfig.getPublicKeyFileName(), overwrite);
+  }
+
+  /**
+   * Helper function that actually writes data to the files.
+   *
+   * @param basePath - base path to write key
+   * @param keyPair - Key pair to write to file.
+   * @param privateKeyFileName - private key file name.
+   * @param publicKeyFileName - public key file name.
+   * @param force - forces overwriting the keys.
+   * @throws IOException - On I/O failure.
+   */
+  private synchronized void writeKey(Path basePath, KeyPair keyPair,
+      String privateKeyFileName, String publicKeyFileName, boolean force)
+      throws IOException {
+    checkPreconditions(basePath);
+
+    File privateKeyFile =
+        Paths.get(location.toString(), privateKeyFileName).toFile();
+    File publicKeyFile =
+        Paths.get(location.toString(), publicKeyFileName).toFile();
+    checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+    try (PemWriter privateKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+      privateKeyWriter.writeObject(
+          new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+    }
+
+    try (PemWriter publicKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+      publicKeyWriter.writeObject(
+          new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+    }
+    Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+    Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+  }
+
+  /**
+   * Checks if private and public key file already exists. Throws IOException
+   * if file exists and force flag is set to false, else will delete the
+   * existing file.
+   *
+   * @param privateKeyFile - Private key file.
+   * @param force - forces overwriting the keys.
+   * @param publicKeyFile - public key file.
+   * @throws IOException - On I/O failure.
+   */
+  private void checkKeyFile(File privateKeyFile, boolean force,
+      File publicKeyFile) throws IOException {
+    if (privateKeyFile.exists() && force) {
+      if (!privateKeyFile.delete()) {
+        throw new IOException("Unable to delete private key file.");
+      }
+    }
+
+    if (publicKeyFile.exists() && force) {
+      if (!publicKeyFile.delete()) {
+        throw new IOException("Unable to delete public key file.");
+      }
+    }
+
+    if (privateKeyFile.exists()) {
+      throw new IOException("Private Key file already exists.");
+    }
+
+    if (publicKeyFile.exists()) {
+      throw new IOException("Public Key file already exists.");
+    }
+  }
+
+  /**
+   * Checks if base path exists and sets file permissions.
+   *
+   * @param basePath - base path to write key
+   * @throws IOException - On I/O failure.
+   */
+  private void checkPreconditions(Path basePath) throws IOException {
+    Preconditions.checkNotNull(basePath, "Base path cannot be null");
+    if (!isPosixFileSystem.get()) {
+      LOG.error("Keys cannot be stored securely without POSIX file system "
+          + "support for now.");
+      throw new IOException("Unsupported File System for pem file.");
+    }
+
+    if (Files.exists(basePath)) {
+      // Not the end of the world if we reset the permissions on an existing
+      // directory.
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    } else {
+      boolean success = basePath.toFile().mkdirs();
+      if (!success) {
+        LOG.error("Unable to create the directory for the "
+            + "location. Location: {}", basePath);
+        throw new IOException("Unable to create the directory for the "
+            + "location. Location:" + basePath);
+      }
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..37a04d6
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Utils for private and public keys.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
deleted file mode 100644
index 2ddf59c..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PublicKey;
-import java.security.interfaces.RSAPublicKey;
-import java.security.spec.PKCS8EncodedKeySpec;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for HDDS Key Generator.
- */
-public class TestHDDSKeyGenerator {
-  private static SecurityConfig config;
-
-  @Before
-  public void init() {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    conf.set(OZONE_METADATA_DIRS,  GenericTestUtils.getTempPath("testpath"));
-    config = new SecurityConfig(conf);
-  }
-  /**
-   * In this test we verify that we are able to create a key pair, then get
-   * bytes of that and use ASN1. parser to parse it back to a private key.
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   */
-  @Test
-  public void testGenerateKey()
-      throws NoSuchProviderException, NoSuchAlgorithmException {
-    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
-    KeyPair keyPair = keyGen.generateKey();
-    Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
-    PKCS8EncodedKeySpec keySpec =
-        new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
-    Assert.assertEquals("PKCS#8", keySpec.getFormat());
-  }
-
-  /**
-   * In this test we assert that size that we specified is used for Key
-   * generation.
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   */
-  @Test
-  public void testGenerateKeyWithSize() throws NoSuchProviderException,
-      NoSuchAlgorithmException {
-    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
-    KeyPair keyPair = keyGen.generateKey(4096);
-    PublicKey publicKey = keyPair.getPublic();
-    if(publicKey instanceof RSAPublicKey) {
-      Assert.assertEquals(4096,
-          ((RSAPublicKey)(publicKey)).getModulus().bitLength());
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
deleted file mode 100644
index 68ff9e6..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test class for HDDS pem writer.
- */
-public class TestHDDSKeyPEMWriter {
-
-  @Rule
-  public TemporaryFolder temporaryFolder = new TemporaryFolder();
-  private OzoneConfiguration configuration;
-  private HDDSKeyGenerator keyGenerator;
-  private String prefix;
-
-  @Before
-  public void init() throws IOException {
-    configuration = new OzoneConfiguration();
-    prefix = temporaryFolder.newFolder().toString();
-    configuration.set(HDDS_METADATA_DIR_NAME, prefix);
-    keyGenerator = new HDDSKeyGenerator(configuration);
-  }
-
-  /**
-   * Assert basic things like we are able to create a file, and the names are
-   * in expected format etc.
-   *
-   * @throws NoSuchProviderException
-   * @throws NoSuchAlgorithmException
-   * @throws IOException
-   */
-  @Test
-  public void testWriteKey()
-      throws NoSuchProviderException, NoSuchAlgorithmException,
-      IOException, InvalidKeySpecException {
-    KeyPair keys = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    pemWriter.writeKey(keys);
-
-    // Assert that locations have been created.
-    Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
-    Assert.assertTrue(keyLocation.toFile().exists());
-
-    // Assert that locations are created in the locations that we specified
-    // using the Config.
-    Assert.assertTrue(keyLocation.toString().startsWith(prefix));
-    Path privateKeyPath = Paths.get(keyLocation.toString(),
-        pemWriter.getSecurityConfig().getPrivateKeyName());
-    Assert.assertTrue(privateKeyPath.toFile().exists());
-    Path publicKeyPath = Paths.get(keyLocation.toString(),
-        pemWriter.getSecurityConfig().getPublicKeyName());
-    Assert.assertTrue(publicKeyPath.toFile().exists());
-
-    // Read the private key and test if the expected String in the PEM file
-    // format exists.
-    byte[] privateKey = Files.readAllBytes(privateKeyPath);
-    String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
-    Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
-
-    // Read the public key and test if the expected String in the PEM file
-    // format exists.
-    byte[] publicKey = Files.readAllBytes(publicKeyPath);
-    String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
-    Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
-
-    // Let us decode the PEM file and parse it back into binary.
-    KeyFactory kf = KeyFactory.getInstance(
-        pemWriter.getSecurityConfig().getAlgo());
-
-    // Replace the PEM Human readable guards.
-    privateKeydata =
-        privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
-    privateKeydata =
-        privateKeydata.replace("-----END PRIVATE KEY-----", "");
-
-    // Decode the bas64 to binary format and then use an ASN.1 parser to
-    // parse the binary format.
-
-    byte[] keyBytes = Base64.decodeBase64(privateKeydata);
-    PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
-    PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
-    Assert.assertNotNull("Private Key should not be null",
-        privateKeyDecoded);
-
-    // Let us decode the public key and veriy that we can parse it back into
-    // binary.
-    publicKeydata =
-        publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
-    publicKeydata =
-        publicKeydata.replace("-----END PUBLIC KEY-----", "");
-
-    keyBytes = Base64.decodeBase64(publicKeydata);
-    X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
-    PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
-    Assert.assertNotNull("Public Key should not be null",
-        publicKeyDecoded);
-
-    // Now let us assert the permissions on the Directories and files are as
-    // expected.
-    Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
-    Set<PosixFilePermission> currentSet =
-        Files.getPosixFilePermissions(privateKeyPath);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-
-    currentSet =
-        Files.getPosixFilePermissions(publicKeyPath);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-
-    currentSet =
-        Files.getPosixFilePermissions(keyLocation);
-    currentSet.removeAll(expectedSet);
-    Assert.assertEquals(0, currentSet.size());
-  }
-
-  /**
-   * Assert key rewrite fails without force option.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testReWriteKey()
-      throws Exception {
-    KeyPair kp = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    SecurityConfig secConfig = pemWriter.getSecurityConfig();
-    pemWriter.writeKey(kp);
-
-    // Assert that rewriting of keys throws exception with valid messages.
-    LambdaTestUtils
-        .intercept(IOException.class, "Private Key file already exists.",
-            () -> pemWriter.writeKey(kp));
-    FileUtils.deleteQuietly(Paths.get(
-        secConfig.getKeyLocation().toString() + "/" + secConfig
-            .getPrivateKeyName()).toFile());
-    LambdaTestUtils
-        .intercept(IOException.class, "Public Key file already exists.",
-            () -> pemWriter.writeKey(kp));
-    FileUtils.deleteQuietly(Paths.get(
-        secConfig.getKeyLocation().toString() + "/" + secConfig
-            .getPublicKeyName()).toFile());
-
-    // Should succeed now as both public and private key are deleted.
-    pemWriter.writeKey(kp);
-    // Should succeed with overwrite flag as true.
-    pemWriter.writeKey(kp, true);
-
-  }
-
-  /**
-   * Assert key rewrite fails in non Posix file system.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testWriteKeyInNonPosixFS()
-      throws Exception {
-    KeyPair kp = keyGenerator.generateKey();
-    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
-    pemWriter.setIsPosixFileSystem(() -> false);
-
-    // Assert key rewrite fails in non Posix file system.
-    LambdaTestUtils
-        .intercept(IOException.class, "Unsupported File System for pem file.",
-            () -> pemWriter.writeKey(kp));
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
new file mode 100644
index 0000000..3a00ca0
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SignatureException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * Test Class for Root Certificate generation.
+ */
+public class TestRootCertificate {
+  private SecurityConfig securityConfig;
+  private static OzoneConfiguration conf = new OzoneConfiguration();
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Before
+  public void init() throws IOException {
+    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+    securityConfig = new SecurityConfig(conf);
+  }
+
+  @Test
+  public void testAllFieldsAreExpected()
+      throws SCMSecurityException, NoSuchProviderException,
+      NoSuchAlgorithmException, CertificateException,
+      SignatureException, InvalidKeyException {
+    Instant now = Instant.now();
+    Date notBefore = Date.from(now);
+    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "testRootCert";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    SelfSignedCertificate.Builder builder =
+        SelfSignedCertificate.newBuilder()
+            .setBeginDate(notBefore)
+            .setEndDate(notAfter)
+            .setClusterID(clusterID)
+            .setScmID(scmID)
+            .setSubject(subject)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+
+    X509CertificateHolder certificateHolder = builder.build();
+
+    //Assert that we indeed have a self signed certificate.
+    Assert.assertEquals(certificateHolder.getIssuer(),
+        certificateHolder.getSubject());
+
+
+    // Make sure that NotBefore is before the current Date
+    Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+    Assert.assertFalse(
+        certificateHolder.getNotBefore()
+            .before(invalidDate));
+
+    //Make sure the end date is honored.
+    invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+    Assert.assertFalse(
+        certificateHolder.getNotAfter()
+            .after(invalidDate));
+
+    // Check the Subject Name and Issuer Name is in the expected format.
+    String dnName = String.format(SelfSignedCertificate.getNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
+    Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
+
+    // We did not ask for this Certificate to be a CA certificate, hence that
+    // extension should be null.
+    Assert.assertNull(
+        certificateHolder.getExtension(Extension.basicConstraints));
+
+    // Extract the Certificate and verify that certificate matches the public
+    // key.
+    X509Certificate cert =
+        new JcaX509CertificateConverter().getCertificate(certificateHolder);
+    cert.verify(keyPair.getPublic());
+  }
+
+  @Test
+  public void testCACert()
+      throws SCMSecurityException, NoSuchProviderException,
+      NoSuchAlgorithmException {
+    Instant now = Instant.now();
+    Date notBefore = Date.from(now);
+    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "testRootCert";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    SelfSignedCertificate.Builder builder =
+        SelfSignedCertificate.newBuilder()
+            .setBeginDate(notBefore)
+            .setEndDate(notAfter)
+            .setClusterID(clusterID)
+            .setScmID(scmID)
+            .setSubject(subject)
+            .setKey(keyPair)
+            .setConfiguration(conf)
+            .makeCA();
+
+    X509CertificateHolder certificateHolder = builder.build();
+    // This time we asked for a CA Certificate, make sure that extension is
+    // present and valid.
+    Extension basicExt =
+        certificateHolder.getExtension(Extension.basicConstraints);
+
+    Assert.assertNotNull(basicExt);
+    Assert.assertTrue(basicExt.isCritical());
+
+    // Since this code assigns ONE for the root certificate, we check if the
+    // serial number is the expected number.
+    Assert.assertEquals(certificateHolder.getSerialNumber(), BigInteger.ONE);
+  }
+
+  @Test
+  public void testInvalidParamFails()
+      throws SCMSecurityException, NoSuchProviderException,
+      NoSuchAlgorithmException {
+    Instant now = Instant.now();
+    Date notBefore = Date.from(now);
+    Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "testRootCert";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    SelfSignedCertificate.Builder builder =
+        SelfSignedCertificate.newBuilder()
+            .setBeginDate(notBefore)
+            .setEndDate(notAfter)
+            .setClusterID(clusterID)
+            .setScmID(scmID)
+            .setSubject(subject)
+            .setConfiguration(conf)
+            .setKey(keyPair)
+            .makeCA();
+    try {
+      builder.setKey(null);
+      builder.build();
+      Assert.fail("Null Key should have failed.");
+    } catch (NullPointerException | IllegalArgumentException e) {
+      builder.setKey(keyPair);
+    }
+
+    // Now try with Blank Subject.
+    try {
+      builder.setSubject("");
+      builder.build();
+      Assert.fail("Null/Blank Subject should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setSubject(subject);
+    }
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setScmID(null);
+      builder.build();
+      Assert.fail("Null/Blank SCM ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setScmID(scmID);
+    }
+
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setClusterID(null);
+      builder.build();
+      Assert.fail("Null/Blank Cluster ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setClusterID(clusterID);
+    }
+
+
+    // Swap the Begin and End Date and verify that we cannot create a
+    // certificate like that.
+    try {
+      builder.setBeginDate(notAfter);
+      builder.setEndDate(notBefore);
+      builder.build();
+      Assert.fail("Illegal dates should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setBeginDate(notBefore);
+      builder.setEndDate(notAfter);
+    }
+
+    try {
+      KeyPair newKey = keyGen.generateKey();
+      KeyPair wrongKey = new KeyPair(newKey.getPublic(), keyPair.getPrivate());
+      builder.setKey(wrongKey);
+      X509CertificateHolder certificateHolder = builder.build();
+      X509Certificate cert =
+          new JcaX509CertificateConverter().getCertificate(certificateHolder);
+      cert.verify(wrongKey.getPublic());
+      Assert.fail("Invalid Key, should have thrown.");
+    } catch (SCMSecurityException | CertificateException
+        | SignatureException | InvalidKeyException e) {
+      builder.setKey(keyPair);
+    }
+    // Assert that we can create a certificate with all sane params.
+    Assert.assertNotNull(builder.build());
+  }
+
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..c8a31fc
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test classes for Certificate utilities.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..f9541a2
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+  private SecurityConfig config;
+
+  @Before
+  public void init() {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OZONE_METADATA_DIRS,  GenericTestUtils.getTempPath("testpath"));
+    config = new SecurityConfig(conf);
+  }
+  /**
+   * In this test we verify that we are able to create a key pair, then get
+   * bytes of that and use ASN1. parser to parse it back to a private key.
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   */
+  @Test
+  public void testGenerateKey()
+      throws NoSuchProviderException, NoSuchAlgorithmException {
+    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+    Assert.assertEquals(config.getKeyAlgo(),
+        keyPair.getPrivate().getAlgorithm());
+    PKCS8EncodedKeySpec keySpec =
+        new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+    Assert.assertEquals("PKCS#8", keySpec.getFormat());
+  }
+
+  /**
+   * In this test we assert that size that we specified is used for Key
+   * generation.
+   * @throws NoSuchProviderException - On Error, due to missing Java
+   * dependencies.
+   * @throws NoSuchAlgorithmException - On Error,  due to missing Java
+   * dependencies.
+   */
+  @Test
+  public void testGenerateKeyWithSize() throws NoSuchProviderException,
+      NoSuchAlgorithmException {
+    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey(4096);
+    PublicKey publicKey = keyPair.getPublic();
+    if(publicKey instanceof RSAPublicKey) {
+      Assert.assertEquals(4096,
+          ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+    }
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[45/50] [abbrv] hadoop git commit: HDDS-873. Fix TestSecureOzoneContainer NPE after HDDS-837. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-873. Fix TestSecureOzoneContainer NPE after HDDS-837. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b9b8e41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b9b8e41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b9b8e41

Branch: refs/heads/HDDS-4
Commit: 9b9b8e410f0d056ec3bae64da1f40013bbefde8b
Parents: 3bee23d
Author: Ajay Kumar <aj...@apache.org>
Authored: Mon Nov 26 15:49:01 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800

----------------------------------------------------------------------
 .../container/ozoneimpl/TestSecureOzoneContainer.java | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b9b8e41/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
index 02d5e28..2224300 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
@@ -45,6 +47,7 @@ import org.junit.rules.TemporaryFolder;
 import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -133,7 +136,7 @@ public class TestSecureOzoneContainer {
           OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
 
       DatanodeDetails dn = TestUtils.randomDatanodeDetails();
-      container = new OzoneContainer(dn, conf, null);
+      container = new OzoneContainer(dn, conf, getContext(dn));
       //Setting scmId, as we start manually ozone container.
       container.getDispatcher().setScmId(UUID.randomUUID().toString());
       container.start();
@@ -206,4 +209,13 @@ public class TestSecureOzoneContainer {
     Assert.assertNotNull(response);
     Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
   }
+
+  private StateContext getContext(DatanodeDetails datanodeDetails) {
+    DatanodeStateMachine stateMachine = Mockito.mock(
+        DatanodeStateMachine.class);
+    StateContext context = Mockito.mock(StateContext.class);
+    Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+    Mockito.when(context.getParent()).thenReturn(stateMachine);
+    return context;
+  }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[29/50] [abbrv] hadoop git commit: HDDS-566. Move OzoneSecure docker-compose after HDDS-447. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-566. Move OzoneSecure docker-compose after HDDS-447. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/585c3448
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/585c3448
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/585c3448

Branch: refs/heads/HDDS-4
Commit: 585c3448684cdae3c9f6c9c175c23038dfef80fc
Parents: eddbe99
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 2 10:07:35 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 hadoop-dist/src/main/compose/ozonesecure/.env   |  18 ---
 .../compose/ozonesecure/docker-compose.yaml     |  57 -------
 .../src/main/compose/ozonesecure/docker-config  | 103 -------------
 .../ozonesecure/docker-image/runner/Dockerfile  |  39 -----
 .../ozonesecure/docker-image/runner/build.sh    |  26 ----
 .../docker-image/runner/scripts/envtoconf.py    | 115 --------------
 .../docker-image/runner/scripts/krb5.conf       |  38 -----
 .../docker-image/runner/scripts/starter.sh      | 100 -------------
 .../runner/scripts/transformation.py            | 150 -------------------
 .../dist/src/main/compose/ozonesecure/.env      |  18 +++
 .../compose/ozonesecure/docker-compose.yaml     |  57 +++++++
 .../src/main/compose/ozonesecure/docker-config  | 103 +++++++++++++
 .../ozonesecure/docker-image/runner/Dockerfile  |  39 +++++
 .../ozonesecure/docker-image/runner/build.sh    |  26 ++++
 .../docker-image/runner/scripts/envtoconf.py    | 115 ++++++++++++++
 .../docker-image/runner/scripts/krb5.conf       |  38 +++++
 .../docker-image/runner/scripts/starter.sh      | 100 +++++++++++++
 .../runner/scripts/transformation.py            | 150 +++++++++++++++++++
 18 files changed, 646 insertions(+), 646 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
deleted file mode 100644
index a494004..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/.env
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
deleted file mode 100644
index 42ab05e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
-   kdc:
-      image: ahadoop/kdc:v1
-      hostname: kdc
-      volumes:
-          - $SRC_VOLUME:/opt/hadoop
-   datanode:
-      image: ahadoop/runner:latest
-      volumes:
-        - $SRC_VOLUME:/opt/hadoop
-      hostname: datanode
-      ports:
-        - 9864
-      command: ["/opt/hadoop/bin/ozone","datanode"]
-      env_file:
-        - ./docker-config
-   ozoneManager:
-      image: ahadoop/runner:latest
-      hostname: om
-      volumes:
-         - $SRC_VOLUME:/opt/hadoop
-      ports:
-         - 9874:9874
-      environment:
-         ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
-      env_file:
-          - ./docker-config
-      command: ["/opt/hadoop/bin/ozone","om"]
-   scm:
-      image: ahadoop/runner:latest
-      hostname: scm
-      volumes:
-         - $SRC_VOLUME:/opt/hadoop
-      ports:
-         - 9876:9876
-      env_file:
-          - ./docker-config
-      environment:
-          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
deleted file mode 100644
index 704dc7b..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-config
+++ /dev/null
@@ -1,103 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=5
-KERBEROS_ENABLED=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
deleted file mode 100644
index efda03e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License lsfor the specific language governing permissions and
-# limitations under the License.
-
-FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y  jq curl python sudo && apt-get clean
-
-RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
-RUN chmod +x /usr/local/bin/dumb-init
-RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-ENV PATH $PATH:/opt/hadoop/bin
-
-RUN apt-get install -y jsvc
-ENV JSVC_HOME=/usr/bin
-ADD scripts/krb5.conf /etc/
-RUN apt-get install -y krb5-user
-
-RUN addgroup --gid 1000 hadoop
-RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
-RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-ADD scripts /opt/
-
-WORKDIR /opt/hadoop
-
-VOLUME /data
-ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
deleted file mode 100755
index abbdebe..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-set -e
-mkdir -p build
-if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
-  wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
-  cd "$DIR"/build || exit
-  tar zvxf apache-rat.tar.gz
-fi
-java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
-docker build -t ahadoop/runner .

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
deleted file mode 100755
index ad2e176..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""convert environment variables to config"""
-
-import os
-import re
-
-import argparse
-
-import sys
-import transformation
-
-class Simple(object):
-  """Simple conversion"""
-  def __init__(self, args):
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--destination", help="Destination directory", required=True)
-    self.args = parser.parse_args(args=args)
-    # copy the default files to file.raw in destination directory
-
-    self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
-    self.output_dir = self.args.destination
-
-    self.configurables = {}
-
-  def destination_file_path(self, name, extension):
-    """destination file path"""
-    return os.path.join(self.output_dir, "{}.{}".format(name, extension))
-
-  def write_env_var(self, name, extension, key, value):
-    """Write environment variables"""
-    with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
-      myfile.write("{}: {}\n".format(key, value))
-
-  def process_envs(self):
-    """Process environment variables"""
-    for key in os.environ.keys():
-      pattern = re.compile("[_\\.]")
-      parts = pattern.split(key)
-      extension = None
-      name = parts[0].lower()
-      if len(parts) > 1:
-        extension = parts[1].lower()
-        config_key = key[len(name) + len(extension) + 2:].strip()
-      if extension and "!" in extension:
-        splitted = extension.split("!")
-        extension = splitted[0]
-        fmt = splitted[1]
-        config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
-      else:
-        fmt = extension
-
-      if extension and extension in self.known_formats:
-        if name not in self.configurables.keys():
-          with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
-            myfile.write("")
-        self.configurables[name] = (extension, fmt)
-        self.write_env_var(name, extension, config_key, os.environ[key])
-      else:
-        for configurable_name in self.configurables:
-          if key.lower().startswith(configurable_name.lower()):
-            self.write_env_var(configurable_name,
-                               self.configurables[configurable_name],
-                               key[len(configurable_name) + 1:],
-                               os.environ[key])
-
-  def transform(self):
-    """transform"""
-    for configurable_name in self.configurables:
-      name = configurable_name
-      extension, fmt = self.configurables[name]
-
-      destination_path = self.destination_file_path(name, extension)
-
-      with open(destination_path + ".raw", "r") as myfile:
-        content = myfile.read()
-        transformer_func = getattr(transformation, "to_" + fmt)
-        content = transformer_func(content)
-        with open(destination_path, "w") as myfile:
-          myfile.write(content)
-
-  def main(self):
-    """main"""
-
-    # add the
-    self.process_envs()
-
-    # copy file.ext.raw to file.ext in the destination directory, and
-    # transform to the right format (eg. key: value ===> XML)
-    self.transform()
-
-
-def main():
-  """main"""
-  Simple(sys.argv[1:]).main()
-
-
-if __name__ == '__main__':
-  Simple(sys.argv[1:]).main()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
deleted file mode 100644
index 82ae73a..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_canonicalize_hostname = false
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- default_realm = EXAMPLE.COM
-
-[realms]
- EXAMPLE.COM = {
-  kdc = SERVER
-  admin_server = SERVER
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
deleted file mode 100755
index 04cd49d..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-set -e
-
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo "Setting up enviorment!!"
-
-if [ -n "$KERBEROS_ENABLED" ]; then
-  echo "Setting up kerberos!!"
-  KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
-  ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
-
-  echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
-
-  if [ -n "$SLEEP_SECONDS" ]; then
-    echo "Sleeping for $(SLEEP_SECONDS) seconds"
-    sleep "$SLEEP_SECONDS"
-  fi
-
-
-  while true
-    do
-      STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
-      if [ "$STATUS" -eq 200 ]; then
-        echo "Got 200, KDC service ready!!"
-        break
-      else
-        echo "Got $STATUS :( KDC service not ready yet..."
-      fi
-      sleep 5
-    done
-
-    HOST_NAME=$(hostname -f)
-    export HOST_NAME
-    for NAME in ${KERBEROS_KEYTABS}; do
-      echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
-      wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
-      klist -kt "$CONF_DIR/$NAME.keytab"
-      KERBEROS_ENABLED=true
-    done
-
-    sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
-fi
-
-#To avoid docker volume permission problems
-sudo chmod o+rwx /data
-
-"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
-
-if [ -n "$ENSURE_NAMENODE_DIR" ]; then
-  CLUSTERID_OPTS=""
-  if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
-    CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
-  fi
-  if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
-    /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
-  fi
-fi
-
-if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
-  if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
-    /opt/hadoop/bin/hdfs namenode -bootstrapStandby
-  fi
-fi
-
-if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
-  if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
-    /opt/hadoop/bin/ozone scm -init
-  fi
-fi
-
-if [ -n "$ENSURE_OM_INITIALIZED" ]; then
-  if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
-    #To make sure SCM is running in dockerized environment we will sleep
-    # Could be removed after HDFS-13203
-    echo "Waiting 15 seconds for SCM startup"
-    sleep 15
-    /opt/hadoop/bin/ozone om -createObjectStore
-  fi
-fi
-
-echo 'setup finished'
-"$@"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
deleted file mode 100755
index 5e708ce..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""This module transform properties into different format"""
-def render_yaml(yaml_root, prefix=""):
-  """render yaml"""
-  result = ""
-  if isinstance(yaml_root, dict):
-    if prefix:
-      result += "\n"
-      for key in yaml_root:
-        result += "{}{}: {}".format(prefix, key, render_yaml(
-            yaml_root[key], prefix + "   "))
-  elif isinstance(yaml_root, list):
-    result += "\n"
-    for item in yaml_root:
-      result += prefix + " - " + render_yaml(item, prefix + " ")
-  else:
-    result += "{}\n".format(yaml_root)
-  return result
-
-
-def to_yaml(content):
-  """transform to yaml"""
-  props = process_properties(content)
-
-  keys = props.keys()
-  yaml_props = {}
-  for key in keys:
-    parts = key.split(".")
-    node = yaml_props
-    prev_part = None
-    parent_node = {}
-    for part in parts[:-1]:
-      if part.isdigit():
-        if isinstance(node, dict):
-          parent_node[prev_part] = []
-          node = parent_node[prev_part]
-        while len(node) <= int(part):
-          node.append({})
-        parent_node = node
-        node = node[int(node)]
-      else:
-        if part not in node:
-          node[part] = {}
-        parent_node = node
-        node = node[part]
-      prev_part = part
-    if parts[-1].isdigit():
-      if isinstance(node, dict):
-        parent_node[prev_part] = []
-        node = parent_node[prev_part]
-      node.append(props[key])
-    else:
-      node[parts[-1]] = props[key]
-
-  return render_yaml(yaml_props)
-
-
-def to_yml(content):
-  """transform to yml"""
-  return to_yaml(content)
-
-
-def to_properties(content):
-  """transform to properties"""
-  result = ""
-  props = process_properties(content)
-  for key, val in props.items():
-    result += "{}: {}\n".format(key, val)
-  return result
-
-
-def to_env(content):
-  """transform to environment variables"""
-  result = ""
-  props = process_properties(content)
-  for key, val in props:
-    result += "{}={}\n".format(key, val)
-  return result
-
-
-def to_sh(content):
-  """transform to shell"""
-  result = ""
-  props = process_properties(content)
-  for key, val in props:
-    result += "export {}=\"{}\"\n".format(key, val)
-  return result
-
-
-def to_cfg(content):
-  """transform to config"""
-  result = ""
-  props = process_properties(content)
-  for key, val in props:
-    result += "{}={}\n".format(key, val)
-  return result
-
-
-def to_conf(content):
-  """transform to configuration"""
-  result = ""
-  props = process_properties(content)
-  for key, val in props:
-    result += "export {}={}\n".format(key, val)
-  return result
-
-
-def to_xml(content):
-  """transform to xml"""
-  result = "<configuration>\n"
-  props = process_properties(content)
-  for key in props:
-    result += "<property><name>{0}</name><value>{1}</value></property>\n". \
-      format(key, props[key])
-  result += "</configuration>"
-  return result
-
-
-def process_properties(content, sep=': ', comment_char='#'):
-  """
-  Read the file passed as parameter as a properties file.
-  """
-  props = {}
-  for line in content.split("\n"):
-    sline = line.strip()
-    if sline and not sline.startswith(comment_char):
-      key_value = sline.split(sep)
-      key = key_value[0].strip()
-      value = sep.join(key_value[1:]).strip().strip('"')
-      props[key] = value
-
-  return props

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   kdc:
+      image: ahadoop/kdc:v1
+      hostname: kdc
+      volumes:
+          - $SRC_VOLUME:/opt/hadoop
+   datanode:
+      image: ahadoop/runner:latest
+      volumes:
+        - $SRC_VOLUME:/opt/hadoop
+      hostname: datanode
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+      env_file:
+        - ./docker-config
+   ozoneManager:
+      image: ahadoop/runner:latest
+      hostname: om
+      volumes:
+         - $SRC_VOLUME:/opt/hadoop
+      ports:
+         - 9874:9874
+      environment:
+         ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","om"]
+   scm:
+      image: ahadoop/runner:latest
+      hostname: scm
+      volumes:
+         - $SRC_VOLUME:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y  jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+  wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+  cd "$DIR"/build || exit
+  tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+  """Simple conversion"""
+  def __init__(self, args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--destination", help="Destination directory", required=True)
+    self.args = parser.parse_args(args=args)
+    # copy the default files to file.raw in destination directory
+
+    self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+    self.output_dir = self.args.destination
+
+    self.configurables = {}
+
+  def destination_file_path(self, name, extension):
+    """destination file path"""
+    return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+  def write_env_var(self, name, extension, key, value):
+    """Write environment variables"""
+    with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+      myfile.write("{}: {}\n".format(key, value))
+
+  def process_envs(self):
+    """Process environment variables"""
+    for key in os.environ.keys():
+      pattern = re.compile("[_\\.]")
+      parts = pattern.split(key)
+      extension = None
+      name = parts[0].lower()
+      if len(parts) > 1:
+        extension = parts[1].lower()
+        config_key = key[len(name) + len(extension) + 2:].strip()
+      if extension and "!" in extension:
+        splitted = extension.split("!")
+        extension = splitted[0]
+        fmt = splitted[1]
+        config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+      else:
+        fmt = extension
+
+      if extension and extension in self.known_formats:
+        if name not in self.configurables.keys():
+          with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+            myfile.write("")
+        self.configurables[name] = (extension, fmt)
+        self.write_env_var(name, extension, config_key, os.environ[key])
+      else:
+        for configurable_name in self.configurables:
+          if key.lower().startswith(configurable_name.lower()):
+            self.write_env_var(configurable_name,
+                               self.configurables[configurable_name],
+                               key[len(configurable_name) + 1:],
+                               os.environ[key])
+
+  def transform(self):
+    """transform"""
+    for configurable_name in self.configurables:
+      name = configurable_name
+      extension, fmt = self.configurables[name]
+
+      destination_path = self.destination_file_path(name, extension)
+
+      with open(destination_path + ".raw", "r") as myfile:
+        content = myfile.read()
+        transformer_func = getattr(transformation, "to_" + fmt)
+        content = transformer_func(content)
+        with open(destination_path, "w") as myfile:
+          myfile.write(content)
+
+  def main(self):
+    """main"""
+
+    # add the
+    self.process_envs()
+
+    # copy file.ext.raw to file.ext in the destination directory, and
+    # transform to the right format (eg. key: value ===> XML)
+    self.transform()
+
+
+def main():
+  """main"""
+  Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+  Simple(sys.argv[1:]).main()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+  kdc = SERVER
+  admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+  echo "Setting up kerberos!!"
+  KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+  ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+  echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+  if [ -n "$SLEEP_SECONDS" ]; then
+    echo "Sleeping for $(SLEEP_SECONDS) seconds"
+    sleep "$SLEEP_SECONDS"
+  fi
+
+
+  while true
+    do
+      STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+      if [ "$STATUS" -eq 200 ]; then
+        echo "Got 200, KDC service ready!!"
+        break
+      else
+        echo "Got $STATUS :( KDC service not ready yet..."
+      fi
+      sleep 5
+    done
+
+    HOST_NAME=$(hostname -f)
+    export HOST_NAME
+    for NAME in ${KERBEROS_KEYTABS}; do
+      echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+      wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+      klist -kt "$CONF_DIR/$NAME.keytab"
+      KERBEROS_ENABLED=true
+    done
+
+    sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+  CLUSTERID_OPTS=""
+  if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+    CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+  fi
+  if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+    /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+  fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+  if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+    /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+  fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+  if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+    /opt/hadoop/bin/ozone scm -init
+  fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+  if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+    #To make sure SCM is running in dockerized environment we will sleep
+    # Could be removed after HDFS-13203
+    echo "Waiting 15 seconds for SCM startup"
+    sleep 15
+    /opt/hadoop/bin/ozone om -createObjectStore
+  fi
+fi
+
+echo 'setup finished'
+"$@"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+  """render yaml"""
+  result = ""
+  if isinstance(yaml_root, dict):
+    if prefix:
+      result += "\n"
+      for key in yaml_root:
+        result += "{}{}: {}".format(prefix, key, render_yaml(
+            yaml_root[key], prefix + "   "))
+  elif isinstance(yaml_root, list):
+    result += "\n"
+    for item in yaml_root:
+      result += prefix + " - " + render_yaml(item, prefix + " ")
+  else:
+    result += "{}\n".format(yaml_root)
+  return result
+
+
+def to_yaml(content):
+  """transform to yaml"""
+  props = process_properties(content)
+
+  keys = props.keys()
+  yaml_props = {}
+  for key in keys:
+    parts = key.split(".")
+    node = yaml_props
+    prev_part = None
+    parent_node = {}
+    for part in parts[:-1]:
+      if part.isdigit():
+        if isinstance(node, dict):
+          parent_node[prev_part] = []
+          node = parent_node[prev_part]
+        while len(node) <= int(part):
+          node.append({})
+        parent_node = node
+        node = node[int(node)]
+      else:
+        if part not in node:
+          node[part] = {}
+        parent_node = node
+        node = node[part]
+      prev_part = part
+    if parts[-1].isdigit():
+      if isinstance(node, dict):
+        parent_node[prev_part] = []
+        node = parent_node[prev_part]
+      node.append(props[key])
+    else:
+      node[parts[-1]] = props[key]
+
+  return render_yaml(yaml_props)
+
+
+def to_yml(content):
+  """transform to yml"""
+  return to_yaml(content)
+
+
+def to_properties(content):
+  """transform to properties"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props.items():
+    result += "{}: {}\n".format(key, val)
+  return result
+
+
+def to_env(content):
+  """transform to environment variables"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "{}={}\n".format(key, val)
+  return result
+
+
+def to_sh(content):
+  """transform to shell"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "export {}=\"{}\"\n".format(key, val)
+  return result
+
+
+def to_cfg(content):
+  """transform to config"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "{}={}\n".format(key, val)
+  return result
+
+
+def to_conf(content):
+  """transform to configuration"""
+  result = ""
+  props = process_properties(content)
+  for key, val in props:
+    result += "export {}={}\n".format(key, val)
+  return result
+
+
+def to_xml(content):
+  """transform to xml"""
+  result = "<configuration>\n"
+  props = process_properties(content)
+  for key in props:
+    result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+      format(key, props[key])
+  result += "</configuration>"
+  return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+  """
+  Read the file passed as parameter as a properties file.
+  """
+  props = {}
+  for line in content.split("\n"):
+    sline = line.strip()
+    if sline and not sline.startswith(comment_char):
+      key_value = sline.split(sep)
+      key = key_value[0].strip()
+      value = sep.join(key_value[1:]).strip().strip('"')
+      props[key] = value
+
+  return props


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[22/50] [abbrv] hadoop git commit: HDDS-100. SCM CA: generate public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-100. SCM CA: generate public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed10fa6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed10fa6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed10fa6f

Branch: refs/heads/HDDS-4
Commit: ed10fa6fb160f210bcf424e151f2eeead34bf184
Parents: c8af727
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Jun 8 08:33:58 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:45 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/pom.xml                      |   6 +-
 .../org/apache/hadoop/hdds/HddsConfigKeys.java  |  19 ++
 .../hdds/security/x509/HDDSKeyGenerator.java    |  99 ++++++++
 .../hdds/security/x509/HDDSKeyPEMWriter.java    | 254 +++++++++++++++++++
 .../hdds/security/x509/SecurityConfig.java      | 190 ++++++++++++++
 .../hadoop/hdds/security/x509/package-info.java |  25 ++
 .../common/src/main/resources/ozone-default.xml |  42 ++-
 .../security/x509/TestHDDSKeyGenerator.java     |  81 ++++++
 .../security/x509/TestHDDSKeyPEMWriter.java     | 213 ++++++++++++++++
 .../ozone/TestOzoneConfigurationFields.java     |   6 +
 10 files changed, 933 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 0345012..175500a 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -81,7 +81,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>rocksdbjni</artifactId>
       <version>5.14.2</version>
     </dependency>
-
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
@@ -110,6 +109,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <version>2.6.0</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk15on</artifactId>
+      <version>1.49</version>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index f16503e..2a1404a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -112,4 +112,23 @@ public final class HddsConfigKeys {
 
   public static final String HDDS_PROMETHEUS_ENABLED =
       "hdds.prometheus.endpoint.enabled";
+
+  public static final String HDDS_KEY_LEN = "hdds.key.len";
+  public static final int HDDS_DEFAULT_KEY_LEN = 2048;
+  public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
+  public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
+  public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
+  public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
+  public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
+  public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
+
+  // TODO : Talk to StorageIO classes and see if they can return a secure
+  // storage location for each node.
+  public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
+  public static final String HDDS_PRIVATE_KEY_FILE_NAME =
+      "hdds.priv.key.file.name";
+  public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
+  public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+      + ".name";
+  public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
new file mode 100644
index 0000000..cb411b2
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(HDDSKeyGenerator.class);
+  private final SecurityConfig securityConfig;
+
+  /**
+   * Constructor for HDDSKeyGenerator.
+   *
+   * @param configuration - config
+   */
+  public HDDSKeyGenerator(Configuration configuration) {
+    this.securityConfig = new SecurityConfig(configuration);
+  }
+
+  /**
+   * Returns the Security config used for this object.
+   * @return SecurityConfig
+   */
+  public SecurityConfig getSecurityConfig() {
+    return securityConfig;
+  }
+
+  /**
+   * Use Config to generate key.
+   *
+   * @return KeyPair
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   */
+  public KeyPair generateKey() throws NoSuchProviderException,
+      NoSuchAlgorithmException {
+    return generateKey(securityConfig.getSize(),
+        securityConfig.getAlgo(), securityConfig.getProvider());
+  }
+
+  /**
+   * Specify the size -- all other parameters are used from config.
+   *
+   * @param size - int, valid key sizes.
+   * @return KeyPair
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   */
+  public KeyPair generateKey(int size) throws
+      NoSuchProviderException, NoSuchAlgorithmException {
+    return generateKey(size,
+        securityConfig.getAlgo(), securityConfig.getProvider());
+  }
+
+  /**
+   * Custom Key Generation, all values are user provided.
+   *
+   * @param size - Key Size
+   * @param algorithm - Algorithm to use
+   * @param provider - Security provider.
+   * @return KeyPair.
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   */
+  public KeyPair generateKey(int size, String algorithm, String provider)
+      throws NoSuchProviderException, NoSuchAlgorithmException {
+    LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+        size, algorithm, provider);
+    KeyPairGenerator generator = KeyPairGenerator
+        .getInstance(algorithm, provider);
+    generator.initialize(size);
+    return generator.generateKeyPair();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..6ca7584
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+  private final Path location;
+  private final SecurityConfig securityConfig;
+  private Set<PosixFilePermission> permissionSet =
+      Stream.of(OWNER_READ, OWNER_WRITE,  OWNER_EXECUTE)
+          .collect(Collectors.toSet());
+  private Supplier<Boolean> isPosixFileSystem;
+  public final static String PRIVATE_KEY = "PRIVATE KEY";
+  public final static String PUBLIC_KEY = "PUBLIC KEY";
+  public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+  /*
+    Creates an HDDS Key Writer.
+
+    @param configuration - Configuration
+   */
+  public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+    Preconditions.checkNotNull(configuration, "Config cannot be null");
+    this.securityConfig = new SecurityConfig(configuration);
+    isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+    this.location = securityConfig.getKeyLocation();
+  }
+
+  /**
+   * Checks if File System supports posix style security permissions.
+   *
+   * @return True if it supports posix.
+   */
+  private static Boolean isPosix() {
+    return FileSystems.getDefault().supportedFileAttributeViews()
+        .contains("posix");
+  }
+
+  /**
+   * Returns the Permission set.
+   * @return Set
+   */
+  @VisibleForTesting
+  public Set<PosixFilePermission> getPermissionSet() {
+    return permissionSet;
+  }
+
+  /**
+   * Returns the Security config used for this object.
+   * @return SecurityConfig
+   */
+  public SecurityConfig getSecurityConfig() {
+    return securityConfig;
+  }
+
+  /**
+   * This function is used only for testing.
+   *
+   * @param isPosixFileSystem - Sets a boolean function for mimicking
+   * files systems that are not posix.
+   */
+  @VisibleForTesting
+  public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+    this.isPosixFileSystem = isPosixFileSystem;
+  }
+
+  /**
+   * Writes a given key using the default config options.
+   *
+   * @param keyPair - Key Pair to write to file.
+   * @throws IOException
+   */
+  public void writeKey(KeyPair keyPair) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+        securityConfig.getPublicKeyName(), false);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException
+   */
+  public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+    writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+        securityConfig.getPublicKeyName(), overwrite);
+  }
+
+  /**
+   * Writes a given key using default config options.
+   *
+   * @param basePath - The location to write to, override the config values.
+   * @param keyPair - Key pair to write
+   * @param overwrite - Overwrites the keys if they already exist.
+   * @throws IOException
+   */
+  public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+      throws IOException {
+    writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
+        securityConfig.getPublicKeyName(), overwrite);
+  }
+
+  /**
+   * Helper function that actually writes data to the files.
+   *
+   * @param basePath - base path to write key
+   * @param keyPair - Key pair to write to file.
+   * @param privateKeyFileName - private key file name.
+   * @param publicKeyFileName - public key file name.
+   * @param force - forces overwriting the keys.
+   * @throws IOException
+   */
+  private synchronized void writeKey(Path basePath, KeyPair keyPair,
+      String privateKeyFileName, String publicKeyFileName, boolean force)
+      throws IOException {
+    checkPreconditions(basePath);
+
+    File privateKeyFile =
+        Paths.get(location.toString(), privateKeyFileName).toFile();
+    File publicKeyFile =
+        Paths.get(location.toString(), publicKeyFileName).toFile();
+    checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+    try (PemWriter privateKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+      privateKeyWriter.writeObject(
+          new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+    }
+
+    try (PemWriter publicKeyWriter = new PemWriter(new
+        FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+      publicKeyWriter.writeObject(
+          new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+    }
+    Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+    Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+  }
+
+  /**
+   * Checks if private and public key file already exists. Throws IOException
+   * if file exists and force flag is set to false, else will delete the
+   * existing file.
+   *
+   * @param privateKeyFile - Private key file.
+   * @param force - forces overwriting the keys.
+   * @param publicKeyFile - public key file.
+   * @throws IOException
+   */
+  private void checkKeyFile(File privateKeyFile, boolean force,
+      File publicKeyFile) throws IOException {
+    if (privateKeyFile.exists() && force) {
+      if (!privateKeyFile.delete()) {
+        throw new IOException("Unable to delete private key file.");
+      }
+    }
+
+    if (publicKeyFile.exists() && force) {
+      if (!publicKeyFile.delete()) {
+        throw new IOException("Unable to delete public key file.");
+      }
+    }
+
+    if (privateKeyFile.exists()) {
+      throw new IOException("Private Key file already exists.");
+    }
+
+    if (publicKeyFile.exists()) {
+      throw new IOException("Public Key file already exists.");
+    }
+  }
+
+  /**
+   * Checks if base path exists and sets file permissions.
+   *
+   * @param basePath - base path to write key
+   * @throws IOException
+   */
+  private void checkPreconditions(Path basePath) throws IOException {
+    Preconditions.checkNotNull(basePath, "Base path cannot be null");
+    if (!isPosixFileSystem.get()) {
+      LOG.error("Keys cannot be stored securely without POSIX file system "
+          + "support for now.");
+      throw new IOException("Unsupported File System for pem file.");
+    }
+
+    if (Files.exists(basePath)) {
+      // Not the end of the world if we reset the permissions on an existing
+      // directory.
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    } else {
+      boolean success = basePath.toFile().mkdirs();
+      if (!success) {
+        LOG.error("Unable to create the directory for the "
+            + "location. Location: {}", basePath);
+        throw new IOException("Unable to create the directory for the "
+            + "location. Location:" + basePath);
+      }
+      Files.setPosixFilePermissions(basePath, permissionSet);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
new file mode 100644
index 0000000..896a379
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.Provider;
+import java.security.Security;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * A class that deals with all Security related configs in HDDDS.
+ * It is easier to have all Java code related to config in a single place.
+ */
+public class SecurityConfig {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SecurityConfig.class);
+  private static volatile Provider provider;
+  private final Configuration configuration;
+  private final int size;
+  private final String algo;
+  private final String providerString;
+  private final String metadatDir;
+  private final String keyDir;
+  private final String privateKeyName;
+  private final String publicKeyName;
+
+  /**
+   * Constructs a HDDSKeyGenerator.
+   *
+   * @param configuration - HDDS Configuration
+   */
+  public SecurityConfig(Configuration configuration) {
+    Preconditions.checkNotNull(configuration, "Configuration cannot be null");
+    this.configuration = configuration;
+    this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
+    this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+        HDDS_DEFAULT_KEY_ALGORITHM);
+    this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
+            HDDS_DEFAULT_SECURITY_PROVIDER);
+
+    // Please Note: To make it easy for our customers we will attempt to read
+    // HDDS metadata dir and if that is not set, we will use Ozone directory.
+    // TODO: We might want to fix this later.
+    this.metadatDir = this.configuration.get(HDDS_METADATA_DIR_NAME,
+        configuration.get(OZONE_METADATA_DIRS));
+
+    Preconditions.checkNotNull(this.metadatDir, "Metadata directory can't be"
+        + " null. Please check configs.");
+    this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
+        HDDS_KEY_DIR_NAME_DEFAULT);
+    this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+        HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
+    this.publicKeyName =  this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+        HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+
+    // First Startup -- if the provider is null, check for the provider.
+    if (SecurityConfig.provider == null) {
+      synchronized (SecurityConfig.class) {
+        provider = Security.getProvider(this.providerString);
+        if (SecurityConfig.provider == null) {
+          // Provider not found, let us try to Dynamically initialize the
+          // provider.
+          provider = initSecurityProvider(this.providerString);
+        }
+      }
+    }
+  }
+
+  /**
+   * Returns the Provider name.
+   * @return String Provider name.
+   */
+  public String getProviderString() {
+    return providerString;
+  }
+
+  /**
+   * Returns the public key file name.
+   * @return String, File name used for public keys.
+   */
+  public String getPublicKeyName() {
+    return publicKeyName;
+  }
+
+  /**
+   * Returns the private key file name.
+   * @return String, File name used for private keys.
+   */
+  public String getPrivateKeyName() {
+    return privateKeyName;
+  }
+
+  /**
+   * Returns the File path to where keys are stored.
+   * @return  String Key location.
+   */
+  public Path getKeyLocation() {
+    return Paths.get(metadatDir, keyDir);
+  }
+
+  /**
+   * Gets the Key Size.
+   *
+   * @return key size.
+   */
+  public int getSize() {
+    return size;
+  }
+
+  /**
+   * Gets provider.
+   *
+   * @return String Provider name.
+   */
+  public String getProvider() {
+    return providerString;
+  }
+
+  /**
+   * Returns the Key generation Algorithm used.
+   *
+   * @return String Algo.
+   */
+  public String getAlgo() {
+    return algo;
+  }
+
+  /**
+   * Returns the Configuration used for initializing this SecurityConfig.
+   * @return  Configuration
+   */
+  public Configuration getConfiguration() {
+    return configuration;
+  }
+
+
+  /**
+   * Adds a security provider dynamically if it is not loaded already.
+   *
+   * @param providerName - name of the provider.
+   */
+  private Provider initSecurityProvider(String providerName) {
+    switch (providerName) {
+    case "BC":
+      Security.addProvider(new BouncyCastleProvider());
+      return Security.getProvider(providerName);
+    default:
+      LOG.error("Security Provider:{} is unknown", provider);
+      throw new SecurityException("Unknown security provider:" + provider);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..89d5d51
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+/**
+ * This package contains common routines used in creating an x509 based
+ * identity framework for HDDS.
+ */
+package org.apache.hadoop.hdds.security.x509;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 13b47c9..0c99368 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1451,4 +1451,44 @@
       OzoneManager http server kerberos keytab.
     </description>
   </property>
-</configuration>
+  <property>
+    <name>hdds.key.len</name>
+    <value>2048</value>
+    <tag>SCM, HDDS, X509, SECURITY</tag>
+    <description>
+      SCM CA key length.  This is an algorithm-specific metric, such as modulus length, specified in number of bits.
+    </description>
+  </property>
+  <property>
+    <name>hdds.key.dir.name</name>
+    <value>keys</value>
+    <tag>SCM, HDDS, X509, SECURITY</tag>
+    <description>
+      Directory to store public/private key for SCM CA. This is relative to ozone/hdds meteadata dir.
+    </description>
+  </property>
+  <property>
+    <name>hdds.metadata.dir</name>
+    <value/>
+    <tag>X509, SECURITY</tag>
+    <description>
+      Absolute path to HDDS metadata dir.
+    </description>
+  </property>
+  <property>
+    <name>hdds.priv.key.file.name</name>
+    <value>private.pem</value>
+    <tag>X509, SECURITY</tag>
+    <description>
+      Name of file which stores private key generated for SCM CA.
+    </description>
+  </property>
+  <property>
+    <name>hdds.public.key.file.name</name>
+    <value>public.pem</value>
+    <tag>X509, SECURITY</tag>
+    <description>
+      Name of file which stores public key generated for SCM CA.
+    </description>
+  </property>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..2ddf59c
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+  private static SecurityConfig config;
+
+  @Before
+  public void init() {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OZONE_METADATA_DIRS,  GenericTestUtils.getTempPath("testpath"));
+    config = new SecurityConfig(conf);
+  }
+  /**
+   * In this test we verify that we are able to create a key pair, then get
+   * bytes of that and use ASN1. parser to parse it back to a private key.
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   */
+  @Test
+  public void testGenerateKey()
+      throws NoSuchProviderException, NoSuchAlgorithmException {
+    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+    Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
+    PKCS8EncodedKeySpec keySpec =
+        new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+    Assert.assertEquals("PKCS#8", keySpec.getFormat());
+  }
+
+  /**
+   * In this test we assert that size that we specified is used for Key
+   * generation.
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   */
+  @Test
+  public void testGenerateKeyWithSize() throws NoSuchProviderException,
+      NoSuchAlgorithmException {
+    HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey(4096);
+    PublicKey publicKey = keyPair.getPublic();
+    if(publicKey instanceof RSAPublicKey) {
+      Assert.assertEquals(4096,
+          ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..68ff9e6
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private OzoneConfiguration configuration;
+  private HDDSKeyGenerator keyGenerator;
+  private String prefix;
+
+  @Before
+  public void init() throws IOException {
+    configuration = new OzoneConfiguration();
+    prefix = temporaryFolder.newFolder().toString();
+    configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+    keyGenerator = new HDDSKeyGenerator(configuration);
+  }
+
+  /**
+   * Assert basic things like we are able to create a file, and the names are
+   * in expected format etc.
+   *
+   * @throws NoSuchProviderException
+   * @throws NoSuchAlgorithmException
+   * @throws IOException
+   */
+  @Test
+  public void testWriteKey()
+      throws NoSuchProviderException, NoSuchAlgorithmException,
+      IOException, InvalidKeySpecException {
+    KeyPair keys = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    pemWriter.writeKey(keys);
+
+    // Assert that locations have been created.
+    Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+    Assert.assertTrue(keyLocation.toFile().exists());
+
+    // Assert that locations are created in the locations that we specified
+    // using the Config.
+    Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+    Path privateKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPrivateKeyName());
+    Assert.assertTrue(privateKeyPath.toFile().exists());
+    Path publicKeyPath = Paths.get(keyLocation.toString(),
+        pemWriter.getSecurityConfig().getPublicKeyName());
+    Assert.assertTrue(publicKeyPath.toFile().exists());
+
+    // Read the private key and test if the expected String in the PEM file
+    // format exists.
+    byte[] privateKey = Files.readAllBytes(privateKeyPath);
+    String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+    // Read the public key and test if the expected String in the PEM file
+    // format exists.
+    byte[] publicKey = Files.readAllBytes(publicKeyPath);
+    String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+    Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+    // Let us decode the PEM file and parse it back into binary.
+    KeyFactory kf = KeyFactory.getInstance(
+        pemWriter.getSecurityConfig().getAlgo());
+
+    // Replace the PEM Human readable guards.
+    privateKeydata =
+        privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+    privateKeydata =
+        privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+    // Decode the bas64 to binary format and then use an ASN.1 parser to
+    // parse the binary format.
+
+    byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+    PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+    PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+    Assert.assertNotNull("Private Key should not be null",
+        privateKeyDecoded);
+
+    // Let us decode the public key and veriy that we can parse it back into
+    // binary.
+    publicKeydata =
+        publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+    publicKeydata =
+        publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+    keyBytes = Base64.decodeBase64(publicKeydata);
+    X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+    PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+    Assert.assertNotNull("Public Key should not be null",
+        publicKeyDecoded);
+
+    // Now let us assert the permissions on the Directories and files are as
+    // expected.
+    Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+    Set<PosixFilePermission> currentSet =
+        Files.getPosixFilePermissions(privateKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(publicKeyPath);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+
+    currentSet =
+        Files.getPosixFilePermissions(keyLocation);
+    currentSet.removeAll(expectedSet);
+    Assert.assertEquals(0, currentSet.size());
+  }
+
+  /**
+   * Assert key rewrite fails without force option.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testReWriteKey()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    SecurityConfig secConfig = pemWriter.getSecurityConfig();
+    pemWriter.writeKey(kp);
+
+    // Assert that rewriting of keys throws exception with valid messages.
+    LambdaTestUtils
+        .intercept(IOException.class, "Private Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPrivateKeyName()).toFile());
+    LambdaTestUtils
+        .intercept(IOException.class, "Public Key file already exists.",
+            () -> pemWriter.writeKey(kp));
+    FileUtils.deleteQuietly(Paths.get(
+        secConfig.getKeyLocation().toString() + "/" + secConfig
+            .getPublicKeyName()).toFile());
+
+    // Should succeed now as both public and private key are deleted.
+    pemWriter.writeKey(kp);
+    // Should succeed with overwrite flag as true.
+    pemWriter.writeKey(kp, true);
+
+  }
+
+  /**
+   * Assert key rewrite fails in non Posix file system.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testWriteKeyInNonPosixFS()
+      throws Exception {
+    KeyPair kp = keyGenerator.generateKey();
+    HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+    pemWriter.setIsPosixFileSystem(() -> false);
+
+    // Assert key rewrite fails in non Posix file system.
+    LambdaTestUtils
+        .intercept(IOException.class, "Unsupported File System for pem file.",
+            () -> pemWriter.writeKey(kp));
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 986618c..7b59903 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -38,5 +38,11 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
     errorIfMissingConfigProps = true;
     errorIfMissingXmlProps = true;
     xmlPropsToSkipCompare.add("hadoop.tags.custom");
+    addPropertiesNotInXml();
+  }
+
+  private void addPropertiesNotInXml() {
+    configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_KEY_ALGORITHM);
+    configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_SECURITY_PROVIDER);
   }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[18/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa7ca153
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa7ca153
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa7ca153

Branch: refs/heads/HDDS-4
Commit: aa7ca153d8a9c9c9381f29f9ed739a56416db015
Parents: ae5fbdd
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800

----------------------------------------------------------------------
 .../authentication/util/KerberosUtil.java       |   2 +-
 .../conf/TestConfigurationFieldsBase.java       |   2 +
 .../java/org/apache/hadoop/hdds/HddsUtils.java  |  13 +-
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |   9 +-
 .../scm/protocol/ScmBlockLocationProtocol.java  |   3 +
 .../StorageContainerLocationProtocol.java       |   4 +
 .../protocolPB/ScmBlockLocationProtocolPB.java  |   6 +
 .../StorageContainerLocationProtocolPB.java     |   4 +
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |   4 +
 .../common/src/main/resources/ozone-default.xml |  26 ++-
 .../StorageContainerDatanodeProtocol.java       |   4 +
 .../StorageContainerDatanodeProtocolPB.java     |   6 +
 .../scm/server/StorageContainerManager.java     |  51 ++++-
 .../StorageContainerManagerHttpServer.java      |   5 +-
 .../hadoop/hdds/scm/block/TestBlockManager.java |   3 +-
 .../ozone/client/protocol/ClientProtocol.java   |   3 +
 hadoop-ozone/common/src/main/bin/start-ozone.sh |  16 +-
 hadoop-ozone/common/src/main/bin/stop-ozone.sh  |  13 +-
 hadoop-ozone/integration-test/pom.xml           |   6 +
 .../TestContainerStateManagerIntegration.java   |   5 +-
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |   4 +-
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |  21 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    | 205 +++++++++++++++++++
 .../ozone/TestStorageContainerManager.java      |   8 +-
 24 files changed, 375 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
index c011045..4459928 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
@@ -167,7 +167,7 @@ public class KerberosUtil {
   }
 
   /* Return fqdn of the current host */
-  static String getLocalHostName() throws UnknownHostException {
+  public static String getLocalHostName() throws UnknownHostException {
     return InetAddress.getLocalHost().getCanonicalHostName();
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 152159b..bce1cd5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,6 +436,8 @@ public abstract class TestConfigurationFieldsBase {
     // Create XML key/value map
     LOG_XML.debug("Reading XML property files\n");
     xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+    // Remove hadoop property set in ozone-default.xml
+    xmlKeyValueMap.remove("hadoop.custom.tags");
     LOG_XML.debug("\n=====\n");
 
     // Create default configuration variable key/value map

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 18637af..fd6a0e3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -278,18 +278,7 @@ public final class HddsUtils {
   }
 
   public static boolean isHddsEnabled(Configuration conf) {
-    String securityEnabled =
-        conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
-            "simple");
-    boolean securityAuthorizationEnabled = conf.getBoolean(
-        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false);
-
-    if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
-      LOG.error("Ozone is not supported in a security enabled cluster. ");
-      return false;
-    } else {
-      return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
-    }
+    return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
   }
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 062b101..e18fe91 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -192,8 +192,9 @@ public final class ScmConfigKeys {
       "ozone.scm.http-address";
   public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
       "ozone.scm.https-address";
-  public static final String OZONE_SCM_KEYTAB_FILE =
-      "ozone.scm.keytab.file";
+  public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
+      "ozone.scm.kerberos.keytab.file";
+  public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
   public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
   public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
   public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
@@ -312,6 +313,10 @@ public final class ScmConfigKeys {
   public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
       "10m";
 
+  public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+      "ozone.scm.web.authentication.kerberos.principal";
+  public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+      "ozone.scm.web.authentication.kerberos.keytab";
   /**
    * Never constructed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index c8d4a80..e17f1c2 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdds.scm.protocol;
 
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -31,6 +33,7 @@ import java.util.List;
  * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
  * to read/write a block.
  */
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ScmBlockLocationProtocol {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index c8a9dcf..5bc2521 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.hdds.scm.protocol;
 
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -27,11 +29,13 @@ import org.apache.hadoop.hdds.protocol.proto
 
 import java.io.IOException;
 import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
  * that currently host a container.
  */
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface StorageContainerLocationProtocol {
   /**
    * Asks SCM where a container should be allocated. SCM responds with the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 837c95b..89bb066 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,9 +18,13 @@
 package org.apache.hadoop.hdds.scm.protocolPB;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
     .ScmBlockLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * Protocol used from an HDFS node to StorageContainerManager.  This extends the
@@ -30,6 +34,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
     "org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
     protocolVersion = 1)
 @InterfaceAudience.Private
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ScmBlockLocationProtocolPB
     extends ScmBlockLocationProtocolService.BlockingInterface {
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index f234ad3..3bd83f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -21,7 +21,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerLocationProtocolProtos
     .StorageContainerLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * Protocol used from an HDFS node to StorageContainerManager.  This extends the
@@ -30,6 +32,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
 @ProtocolInfo(protocolName =
     "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
     protocolVersion = 1)
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface StorageContainerLocationProtocolPB
     extends StorageContainerLocationProtocolService.BlockingInterface {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index df233f7..b47113b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -328,6 +328,10 @@ public final class OzoneConfigKeys {
   public static final double
       HDDS_DATANODE_STORAGE_UTILIZATION_CRITICAL_THRESHOLD_DEFAULT = 0.75;
 
+  public static final String OZONE_SECURITY_ENABLED_KEY =
+      "ozone.security.enabled";
+  public static final boolean OZONE_SECURITY_ENABLED_DEFAULT = false;
+
   public static final String OZONE_CONTAINER_COPY_WORKDIR =
       "hdds.datanode.replication.work.dir";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0545805..a51443c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1277,7 +1277,6 @@
       datanode unless the datanode confirms the completion.
     </description>
   </property>
-
   <property>
     <name>hdds.db.profile</name>
     <value>DISK</value>
@@ -1286,7 +1285,6 @@
     that tunes the RocksDB settings for the hardware it is running
     on. Right now, we have SSD and DISK as profile options.</description>
   </property>
-
   <property>
     <name>hdds.datanode.replication.work.dir</name>
     <tag>DATANODE</tag>
@@ -1389,5 +1387,29 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.scm.kerberos.keytab.file</name>
+    <value></value>
+    <tag> OZONE, SECURITY</tag>
+    <description> The keytab file used by each SCM daemon to login as its
+      service principal. The principal name is configured with
+      ozone.scm.kerberos.principal.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.kerberos.principal</name>
+    <value></value>
+    <tag> OZONE, SECURITY</tag>
+    <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+  </property>
+
+  <property>
+    <name>ozone.scm.web.authentication.kerberos.principal</name>
+    <value>HTTP/_HOST@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>ozone.scm.web.authentication.kerberos.keytab</name>
+    <value>/etc/security/keytabs/HTTP.keytab</value>
+  </property>
 
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index e3b5370..8049e9d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -36,11 +36,15 @@ import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
 
 import java.io.IOException;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * The protocol spoken between datanodes and SCM. For specifics please the
  * Protoc file that defines this protocol.
  */
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 @InterfaceAudience.Private
 public interface StorageContainerDatanodeProtocol {
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9b28b5a..9c32ef8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -19,7 +19,10 @@ package org.apache.hadoop.ozone.protocolPB;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos
     .StorageContainerDatanodeProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * Protocol used from a datanode to StorageContainerManager.  This extends
@@ -29,6 +32,9 @@ import org.apache.hadoop.ipc.ProtocolInfo;
 @ProtocolInfo(protocolName =
     "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
     protocolVersion = 1)
+@KerberosInfo(
+    serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+    clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
 public interface StorageContainerDatanodeProtocolPB extends
     StorageContainerDatanodeProtocolService.BlockingInterface {
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2d27984..519f51b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -28,10 +28,12 @@ import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
 import com.google.protobuf.BlockingService;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.hdds.scm.block.BlockManager;
 import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
 import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
@@ -82,6 +84,9 @@ import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.common.StorageInfo;
 import org.apache.hadoop.ozone.lease.LeaseManager;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.StringUtils;
 
@@ -101,6 +106,10 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
 /**
@@ -189,10 +198,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
    *
    * @param conf configuration
    */
-  private StorageContainerManager(OzoneConfiguration conf) throws IOException {
+  private StorageContainerManager(OzoneConfiguration conf)
+      throws IOException, AuthenticationException {
 
     StorageContainerManager.initMetrics();
     initContainerReportCache(conf);
+    // Authenticate SCM if security is enabled
+    if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+        OZONE_SECURITY_ENABLED_DEFAULT)) {
+      loginAsSCMUser(conf);
+    }
 
     scmStorage = new SCMStorage(conf);
     if (scmStorage.getState() != StorageState.INITIALIZED) {
@@ -310,6 +325,33 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
   }
 
   /**
+   * Login as the configured user for SCM.
+   *
+   * @param conf
+   */
+  private void loginAsSCMUser(Configuration conf)
+      throws IOException, AuthenticationException {
+    LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
+            + "Principal: {}, keytab: {}", conf.get
+            (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
+        conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+
+    if (SecurityUtil.getAuthenticationMethod(conf).equals
+        (AuthenticationMethod.KERBEROS)) {
+      UserGroupInformation.setConfiguration(conf);
+      InetSocketAddress socAddr = HddsServerUtil
+          .getScmBlockClientBindAddress(conf);
+      SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+          OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+    } else {
+      throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+          conf) + " authentication method not support. "
+          + "SCM user login failed.");
+    }
+    LOG.info("SCM login successful.");
+  }
+
+  /**
    * Builds a message for logging startup information about an RPC server.
    *
    * @param description RPC server description
@@ -403,8 +445,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
    * @throws IOException
    */
   @VisibleForTesting
-  public static StorageContainerManager createSCM(
-      String[] args, OzoneConfiguration conf) throws IOException {
+  public static StorageContainerManager createSCM(String[] args,
+      OzoneConfiguration conf) throws IOException, AuthenticationException {
     return createSCM(args, conf, false);
   }
 
@@ -420,7 +462,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
   private static StorageContainerManager createSCM(
       String[] args,
       OzoneConfiguration conf,
-      boolean printBanner) throws IOException {
+      boolean printBanner)
+      throws IOException, AuthenticationException {
     String[] argv = (args == null) ? new String[0] : args;
     if (!HddsUtils.isHddsEnabled(conf)) {
       System.err.println(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 75b2036..da936ad 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.server;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.server.BaseHttpServer;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 
 import java.io.IOException;
 
@@ -63,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
   }
 
   @Override protected String getKeytabFile() {
-    return ScmConfigKeys.OZONE_SCM_KEYTAB_FILE;
+    return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
   }
 
   @Override protected String getSpnegoPrincipal() {
-    return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+    return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
   }
 
   @Override protected String getEnabledKey() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index ce7fa2f..71a0144 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Assert;
@@ -113,7 +114,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
   }
 
   private static StorageContainerManager getScm(OzoneConfiguration conf)
-      throws IOException {
+      throws IOException, AuthenticationException {
     conf.setBoolean(OZONE_ENABLED, true);
     SCMStorage scmStore = new SCMStorage(conf);
     if(scmStore.getState() != StorageState.INITIALIZED) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index be5670d..f5b2cd9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.client.protocol;
 
 import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.client.*;
 import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 
 import java.io.IOException;
 import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * An implementer of this interface is capable of connecting to Ozone Cluster
@@ -38,6 +40,7 @@ import java.util.List;
  * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
  * {@link  org.apache.hadoop.ozone.client.rest.RestClient} for REST.
  */
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
 public interface ClientProtocol {
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index 4c022fb..a05b9ae 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -70,10 +70,18 @@ nameStartOpt="$nameStartOpt $*"
 SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
 SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
 
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
-  echo "Ozone is not supported in a security enabled cluster."
-  exit 1
-fi
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED}
+# == "true" ]]; then
+#  echo "Ozone is not supported in a security enabled cluster."
+#  exit 1
+#fi
+
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+#  echo "Ozone is not supported in a security enabled cluster."
+#  exit 1
+#fi
 
 #---------------------------------------------------------
 # Check if ozone is enabled

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index fe902eb..c07d42b 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,13 +47,12 @@ else
   exit 1
 fi
 
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
-  echo "Ozone is not supported in a security enabled cluster."
-  exit 1
-fi
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+#  echo "Ozone is not supported in a security enabled cluster."
+#  exit 1
+#fi
 
 #---------------------------------------------------------
 # Check if ozone is enabled

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index 67a166c..1867d95 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -40,6 +40,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minikdc</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-objectstore-service</artifactId>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
index 15ee6f6..0573b38 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -107,8 +108,8 @@ public class TestContainerStateManagerIntegration {
   }
 
   @Test
-  public void testContainerStateManagerRestart()
-      throws IOException, TimeoutException, InterruptedException {
+  public void testContainerStateManagerRestart() throws IOException,
+      TimeoutException, InterruptedException, AuthenticationException {
     // Allocate 5 containers in ALLOCATED state and 5 in CREATING state
 
     for (int i = 0; i < 10; i++) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index b352e36..3fe5b76 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.test.GenericTestUtils;
 
 import java.io.IOException;
@@ -149,8 +150,7 @@ public interface MiniOzoneCluster {
    * @throws TimeoutException
    * @throws InterruptedException
    */
-  void restartStorageContainerManager() throws InterruptedException,
-      TimeoutException, IOException;
+  void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
 
   /**
    * Restarts OzoneManager instance.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..7b0d73d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.protocolPB
 import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.test.GenericTestUtils;
 
 import org.slf4j.Logger;
@@ -232,8 +233,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
   }
 
   @Override
-  public void restartStorageContainerManager()
-      throws TimeoutException, InterruptedException, IOException {
+  public void restartStorageContainerManager() throws TimeoutException,
+      InterruptedException, IOException, AuthenticationException {
     scm.stop();
     scm.join();
     scm = StorageContainerManager.createSCM(null, conf);
@@ -369,9 +370,16 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
     public MiniOzoneCluster build() throws IOException {
       DefaultMetricsSystem.setMiniClusterMode(true);
       initializeConfiguration();
-      StorageContainerManager scm = createSCM();
-      scm.start();
-      OzoneManager om = createOM();
+      StorageContainerManager scm;
+      OzoneManager om;
+      try {
+        scm = createSCM();
+        scm.start();
+        om = createOM();
+      } catch (AuthenticationException ex) {
+        throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+      }
+
       om.start();
       final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
       MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
@@ -423,7 +431,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
      *
      * @throws IOException
      */
-    private StorageContainerManager createSCM() throws IOException {
+    private StorageContainerManager createSCM()
+        throws IOException, AuthenticationException {
       configureSCM();
       SCMStorage scmStore = new SCMStorage(conf);
       initializeScmStorage(scmStore);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
new file mode 100644
index 0000000..9c430ad
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+import java.util.UUID;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.server.SCMStorage;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class to for security enabled Ozone cluster.
+ */
+@InterfaceAudience.Private
+public final class TestSecureOzoneCluster {
+
+  private Logger LOGGER = LoggerFactory
+      .getLogger(TestSecureOzoneCluster.class);
+
+  private MiniKdc miniKdc;
+  private OzoneConfiguration conf;
+  private File workDir;
+  private static Properties securityProperties;
+  private File scmKeytab;
+  private File spnegoKeytab;
+  private String curUser;
+
+  @Before
+  public void init() {
+    try {
+      conf = new OzoneConfiguration();
+      startMiniKdc();
+      setSecureConfig(conf);
+      createCredentialsInKDC(conf, miniKdc);
+    } catch (IOException e) {
+      LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+    } catch (Exception e) {
+      LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+    }
+  }
+
+  private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
+      throws Exception {
+    createPrincipal(scmKeytab,
+        conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
+    createPrincipal(spnegoKeytab,
+        conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+  }
+
+  private void createPrincipal(File keytab, String... principal)
+      throws Exception {
+    miniKdc.createPrincipal(keytab, principal);
+  }
+
+  private void startMiniKdc() throws Exception {
+    workDir = GenericTestUtils
+        .getTestDir(TestSecureOzoneCluster.class.getSimpleName());
+    securityProperties = MiniKdc.createConf();
+    miniKdc = new MiniKdc(securityProperties, workDir);
+    miniKdc.start();
+  }
+
+  private void setSecureConfig(Configuration conf) throws IOException {
+    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+    String host = KerberosUtil.getLocalHostName();
+    String realm = miniKdc.getRealm();
+    curUser = UserGroupInformation.getCurrentUser()
+        .getUserName();
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "kerberos");
+    conf.set(OZONE_ADMINISTRATORS, curUser);
+
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+        "scm/" + host + "@" + realm);
+    conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+        "HTTP_SCM/" + host + "@" + realm);
+
+    scmKeytab = new File(workDir, "scm.keytab");
+    spnegoKeytab = new File(workDir, "http.keytab");
+
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+        scmKeytab.getAbsolutePath());
+    conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+        spnegoKeytab.getAbsolutePath());
+
+  }
+
+  @Test
+  public void testSecureScmStartupSuccess() throws Exception {
+    final String path = GenericTestUtils
+        .getTempPath(UUID.randomUUID().toString());
+    Path scmPath = Paths.get(path, "scm-meta");
+    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+    conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+    SCMStorage scmStore = new SCMStorage(conf);
+    String clusterId = UUID.randomUUID().toString();
+    String scmId = UUID.randomUUID().toString();
+    scmStore.setClusterId(clusterId);
+    scmStore.setScmId(scmId);
+    // writes the version file properties
+    scmStore.initialize();
+    StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
+    //Reads the SCM Info from SCM instance
+    ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+    Assert.assertEquals(clusterId, scmInfo.getClusterId());
+    Assert.assertEquals(scmId, scmInfo.getScmId());
+  }
+
+  @Test
+  public void testSecureScmStartupFailure() throws Exception {
+    final String path = GenericTestUtils
+        .getTempPath(UUID.randomUUID().toString());
+    Path scmPath = Paths.get(path, "scm-meta");
+
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+    conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+        "scm@" + miniKdc.getRealm());
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "kerberos");
+
+    SCMStorage scmStore = new SCMStorage(conf);
+    String clusterId = UUID.randomUUID().toString();
+    String scmId = UUID.randomUUID().toString();
+    scmStore.setClusterId(clusterId);
+    scmStore.setScmId(scmId);
+    // writes the version file properties
+    scmStore.initialize();
+    LambdaTestUtils.intercept(IOException.class,
+        "Running in secure mode, but config doesn't have a keytab",
+        () -> {
+          StorageContainerManager.createSCM(null, conf);
+        });
+
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+        "scm/_HOST@EXAMPLE.com");
+    conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+        "/etc/security/keytabs/scm.keytab");
+
+    LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
+            + "to login: for principal:",
+        () -> {
+          StorageContainerManager.createSCM(null, conf);
+        });
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "OAuth2");
+
+    LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+            + " attribute value for hadoop.security.authentication of OAuth2",
+        () -> {
+          StorageContainerManager.createSCM(null, conf);
+        });
+
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "KERBEROS_SSL");
+    LambdaTestUtils.intercept(AuthenticationException.class,
+        "KERBEROS_SSL authentication method not support.",
+        () -> {
+          StorageContainerManager.createSCM(null, conf);
+        });
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 23d371d..c79da17 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -59,8 +59,10 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.ExitUtil;
+
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -426,7 +428,8 @@ public class TestStorageContainerManager {
   }
 
   @Test
-  public void testSCMInitializationFailure() throws IOException {
+  public void testSCMInitializationFailure()
+      throws IOException, AuthenticationException {
     OzoneConfiguration conf = new OzoneConfiguration();
     final String path =
         GenericTestUtils.getTempPath(UUID.randomUUID().toString());
@@ -439,7 +442,8 @@ public class TestStorageContainerManager {
   }
 
   @Test
-  public void testSCMInitializationReturnCode() throws IOException {
+  public void testSCMInitializationReturnCode() throws IOException,
+      AuthenticationException {
     ExitUtil.disableSystemExit();
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[20/50] [abbrv] hadoop git commit: HDDS-7. Enable kerberos auth for Ozone client in hadoop rpc. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-7. Enable kerberos auth for Ozone client in hadoop rpc. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c821937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c821937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c821937

Branch: refs/heads/HDDS-4
Commit: 5c821937a4972c9a6001c18e18b7a58891198011
Parents: 074b8f4
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri May 18 13:09:17 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800

----------------------------------------------------------------------
 .../src/test/compose/compose-secure/.env        | 17 ++++
 .../compose/compose-secure/docker-compose.yaml  | 66 ++++++++++++++
 .../test/compose/compose-secure/docker-config   | 66 ++++++++++++++
 .../acceptance/ozone-secure.robot               | 95 ++++++++++++++++++++
 .../hadoop/ozone/client/rest/RestClient.java    |  4 +-
 .../hadoop/ozone/client/rpc/RpcClient.java      |  6 +-
 6 files changed, 248 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
new file mode 100644
index 0000000..3254735
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
new file mode 100644
index 0000000..2661163
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   ozone.kdc:
+      image: ahadoop/kdc:v1
+   namenode:
+      image: ahadoop/ozone:v1
+      hostname: namenode
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9000:9000
+      environment:
+          ENSURE_NAMENODE_DIR: /data/namenode
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","namenode"]
+   datanode:
+      image: ahadoop/ozone:v1
+      hostname: datanode
+      volumes:
+        - ${OZONEDIR}:/opt/hadoop
+      ports:
+        - 9874
+      env_file:
+        - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+   ksm:
+      image: ahadoop/ozone:v1
+      hostname: ksm
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9874:9874
+      environment:
+         ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","ksm"]
+   scm:
+      image: ahadoop/ozone:v1
+      hostname: scm
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
new file mode 100644
index 0000000..678c75a
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_hdds.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_hdds.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+HDFS-SITE.XML_dfs.block.access.token.enable=true
+HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=ozone.kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=10
+KERBEROS_ENABLED=true

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
new file mode 100644
index 0000000..4a78980
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test to start cluster with docker-compose environments.
+Library             OperatingSystem
+Suite Setup         Startup Ozone Cluster
+Suite Teardown      Teardown Ozone Cluster
+
+*** Variables ***
+${COMMON_REST_HEADER}   -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H  "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
+${version}
+
+*** Test Cases ***
+
+Daemons are running
+    Is daemon running           ksm
+    Is daemon running           scm
+    Is daemon running           datanode
+    Is daemon running           ozone.kdc
+
+Check if datanode is connected to the scm
+    Wait Until Keyword Succeeds     3min    5sec    Have healthy datanodes   1
+
+Test rest interface
+    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+                    Should contain      ${result}       201 Created
+    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+                    Should contain      ${result}       201 Created
+    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+                    Should contain      ${result}       200 OK
+    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+                    Should contain      ${result}       200 OK
+
+Test ozone cli
+    ${result} =     Execute on      1   datanode        ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+                    Should contain      ${result}       Client cannot authenticate via
+                    # Authenticate testuser
+                    Execute on      0   datanode        kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+                    Execute on      0   datanode        ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+    ${result} =     Execute on      0   datanode        ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+                    Should contain      ${result}       createdOn
+                    Execute on      0   datanode        ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
+    ${result} =     Execute on      0   datanode        ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+                    Should Be Equal     ${result}       bill
+
+*** Keywords ***
+
+Startup Ozone Cluster
+    ${rc}       ${output} =                 Run docker compose        0     down
+    ${rc}       ${output} =                 Run docker compose        0     up -d
+    Should Be Equal As Integers 	          ${rc} 	                  0
+    Wait Until Keyword Succeeds             3min    10sec    Is Daemon started   ksm     KSM is listening
+
+Teardown Ozone Cluster
+    Run docker compose      0           down
+
+Is daemon running
+    [arguments]             ${name}
+    ${result} =             Run                     docker ps
+    Should contain          ${result}               _${name}_1
+
+Is Daemon started
+    [arguments]     ${name}             ${expression}
+    ${rc}           ${result} =         Run docker compose      0         logs
+    Should contain  ${result}           ${expression}
+
+Have healthy datanodes
+    [arguments]         ${requirednodes}
+    ${result} =         Execute on     0     scm                 curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
+    Should Be Equal     ${result}           ${requirednodes}
+
+Execute on
+    [arguments]     ${expected_rc}      ${componentname}      ${command}
+    ${rc}           ${return} =         Run docker compose    ${expected_rc}     exec ${componentname} ${command}
+    [return]        ${return}
+
+Run docker compose
+    [arguments]                     ${expected_rc}              ${command}
+                                    Set Environment Variable    OZONEDIR                               ${basedir}/hadoop-dist/target/ozone
+    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
+    Should Be Equal As Integers     ${rc}                       ${expected_rc}
+    [return]                        ${rc}                       ${output}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 8889d99..3fd0927 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -111,7 +111,7 @@ public class RestClient implements ClientProtocol {
     try {
       Preconditions.checkNotNull(conf);
       this.conf = conf;
-
+      this.ugi = UserGroupInformation.getCurrentUser();
       long socketTimeout = conf.getTimeDuration(
           OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
           OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
@@ -147,7 +147,7 @@ public class RestClient implements ClientProtocol {
                   .setConnectTimeout(Math.toIntExact(connectionTimeout))
                   .build())
           .build();
-      this.ugi = UserGroupInformation.getCurrentUser();
+
       this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
           OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 65adbfa..1b70f3b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -126,8 +126,7 @@ public class RpcClient implements ClientProtocol {
     this.ozoneManagerClient =
         new OzoneManagerProtocolClientSideTranslatorPB(
             RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
-                omAddress, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
+                omAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
                 Client.getRpcTimeout(conf)));
 
     long scmVersion =
@@ -138,8 +137,7 @@ public class RpcClient implements ClientProtocol {
     this.storageContainerLocationClient =
         new StorageContainerLocationProtocolClientSideTranslatorPB(
             RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
-                scmAddress, UserGroupInformation.getCurrentUser(), conf,
-                NetUtils.getDefaultSocketFactory(conf),
+                scmAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
                 Client.getRpcTimeout(conf)));
 
     this.xceiverClientManager = new XceiverClientManager(conf);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[24/50] [abbrv] hadoop git commit: HDDS-684. Fix HDDS-4 branch after HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-684. Fix HDDS-4 branch after HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72468986
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72468986
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72468986

Branch: refs/heads/HDDS-4
Commit: 724689864b6723a2ada9ccb925f9b828c19f305b
Parents: 950a473
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 24 15:46:34 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 hadoop-ozone/dist/dev-support/bin/dist-layout-stitching |  2 +-
 .../dist/src/main/compose/ozonesecure/docker-config     |  1 +
 .../ozonesecure/docker-image/runner/scripts/starter.sh  |  4 ++--
 .../java/org/apache/hadoop/ozone/om/OzoneManager.java   | 12 ++++++------
 4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index e736065..2a4f0ad 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -111,5 +111,5 @@ run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore
 cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./
 
 #Copy docker compose files
-run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" .
+run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/compose" .
 run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .

http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index 704dc7b..36f05ae 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -37,6 +37,7 @@ HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
 HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
 HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
 HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
 CORE-SITE.XML_dfs.data.transfer.protection=authentication

http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
index 04cd49d..eec7ce9 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -82,7 +82,7 @@ fi
 
 if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
   if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
-    /opt/hadoop/bin/ozone scm -init
+    /opt/hadoop/bin/ozone scm --init
   fi
 fi
 
@@ -92,7 +92,7 @@ if [ -n "$ENSURE_OM_INITIALIZED" ]; then
     # Could be removed after HDFS-13203
     echo "Waiting 15 seconds for SCM startup"
     sleep 15
-    /opt/hadoop/bin/ozone om -createObjectStore
+    /opt/hadoop/bin/ozone om --init
   fi
 fi
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 7fc6aee..a5cec20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -259,17 +259,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
 
 
   /**
-   * Login KSM service user if security and Kerberos are enabled.
+   * Login OM service user if security and Kerberos are enabled.
    *
    * @param  conf
    * @throws IOException, AuthenticationException
    */
-  private static void loginKSMUser(OzoneConfiguration conf)
+  private static void loginOMUser(OzoneConfiguration conf)
       throws IOException, AuthenticationException {
 
     if (SecurityUtil.getAuthenticationMethod(conf).equals
         (AuthenticationMethod.KERBEROS)) {
-      LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+      LOG.debug("Ozone security is enabled. Attempting login for OM user. "
               + "Principal: {},keytab: {}", conf.get(
           OZONE_OM_KERBEROS_PRINCIPAL_KEY),
           conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
@@ -281,10 +281,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
           OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
     } else {
       throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
-          (conf) + " authentication method not supported. KSM user login "
+          (conf) + " authentication method not supported. OM user login "
           + "failed.");
     }
-    LOG.info("KSM login successful.");
+    LOG.info("Ozone Manager login successful.");
   }
 
   /**
@@ -440,7 +440,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
     }
     // Authenticate KSM if security is enabled
     if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
-      loginKSMUser(conf);
+      loginOMUser(conf);
     }
     switch (startOpt) {
     case INIT:


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[19/50] [abbrv] hadoop git commit: Revert "Bad merge with 996a627b289947af3894bf83e7b63ec702a665cd"

Posted by xy...@apache.org.
Revert "Bad merge with 996a627b289947af3894bf83e7b63ec702a665cd"

This reverts commit 996a627b289947af3894bf83e7b63ec702a665cd.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/074b8f4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/074b8f4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/074b8f4e

Branch: refs/heads/HDDS-4
Commit: 074b8f4ed52a9defaedaff788c517d7b3f3dd024
Parents: fcd705a
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 15 16:56:24 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ----
 1 file changed, 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/074b8f4e/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4d79e8c..d3e352b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -198,7 +198,6 @@
     </description>
   </property>
   <property>
-<<<<<<< HEAD
     <name>dfs.ratis.client.request.timeout.duration</name>
     <value>3s</value>
     <tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -256,9 +255,6 @@
   </property>
   <property>
     <name>hdds.container.report.interval</name>
-=======
-    <name>ozone.container.report.interval</name>
->>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
     <value>60000ms</value>
     <tag>OZONE, CONTAINER, MANAGEMENT</tag>
     <description>Time interval of the datanode to send container report. Each


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[07/50] [abbrv] hadoop git commit: HDFS-13713. Add specification of Multipart Upload API to FS specification, with contract tests.

Posted by xy...@apache.org.
HDFS-13713. Add specification of Multipart Upload API to FS specification, with contract tests.

Contributed by Ewan Higgs and Steve Loughran.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1d24f84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1d24f84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1d24f84

Branch: refs/heads/HDDS-4
Commit: c1d24f848345f6d34a2ac2d570d49e9787a0df6a
Parents: b71cc7f
Author: Ewan Higgs <eh...@apache.org>
Authored: Thu Nov 29 15:11:07 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 15:12:17 2018 +0000

----------------------------------------------------------------------
 .../hadoop/fs/FileSystemMultipartUploader.java  |  36 +-
 .../org/apache/hadoop/fs/MultipartUploader.java |  88 ++-
 .../hadoop/fs/MultipartUploaderFactory.java     |   7 +
 .../src/site/markdown/filesystem/index.md       |   1 +
 .../markdown/filesystem/multipartuploader.md    | 235 ++++++++
 .../AbstractContractMultipartUploaderTest.java  | 565 +++++++++++++++----
 .../TestLocalFSContractMultipartUploader.java   |  10 +
 .../hdfs/TestHDFSContractMultipartUploader.java |  15 +
 .../hadoop/fs/s3a/S3AMultipartUploader.java     |  31 +-
 .../s3a/ITestS3AContractMultipartUploader.java  |  64 ++-
 .../fs/s3a/TestS3AMultipartUploaderSupport.java |   2 +-
 11 files changed, 876 insertions(+), 178 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 94c7861..b77c244 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -19,21 +19,23 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
 
 import org.apache.commons.compress.utils.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 
 import static org.apache.hadoop.fs.Path.mergePaths;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
 
 /**
  * A MultipartUploader that uses the basic FileSystem commands.
@@ -70,7 +72,8 @@ public class FileSystemMultipartUploader extends MultipartUploader {
   public PartHandle putPart(Path filePath, InputStream inputStream,
       int partNumber, UploadHandle uploadId, long lengthInBytes)
       throws IOException {
-
+    checkPutArguments(filePath, inputStream, partNumber, uploadId,
+        lengthInBytes);
     byte[] uploadIdByteArray = uploadId.toByteArray();
     checkUploadId(uploadIdByteArray);
     Path collectorPath = new Path(new String(uploadIdByteArray, 0,
@@ -82,16 +85,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
             fs.createFile(partPath).build()) {
       IOUtils.copy(inputStream, fsDataOutputStream, 4096);
     } finally {
-      org.apache.hadoop.io.IOUtils.cleanupWithLogger(LOG, inputStream);
+      cleanupWithLogger(LOG, inputStream);
     }
     return BBPartHandle.from(ByteBuffer.wrap(
         partPath.toString().getBytes(Charsets.UTF_8)));
   }
 
   private Path createCollectorPath(Path filePath) {
+    String uuid = UUID.randomUUID().toString();
     return mergePaths(filePath.getParent(),
         mergePaths(new Path(filePath.getName().split("\\.")[0]),
-            mergePaths(new Path("_multipart"),
+            mergePaths(new Path("_multipart_" + uuid),
                 new Path(Path.SEPARATOR))));
   }
 
@@ -110,21 +114,16 @@ public class FileSystemMultipartUploader extends MultipartUploader {
 
   @Override
   @SuppressWarnings("deprecation") // rename w/ OVERWRITE
-  public PathHandle complete(Path filePath,
-      List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
-      throws IOException {
+  public PathHandle complete(Path filePath, Map<Integer, PartHandle> handleMap,
+      UploadHandle multipartUploadId) throws IOException {
 
     checkUploadId(multipartUploadId.toByteArray());
 
-    if (handles.isEmpty()) {
-      throw new IOException("Empty upload");
-    }
-    // If destination already exists, we believe we already completed it.
-    if (fs.exists(filePath)) {
-      return getPathHandle(filePath);
-    }
+    checkPartHandles(handleMap);
+    List<Map.Entry<Integer, PartHandle>> handles =
+        new ArrayList<>(handleMap.entrySet());
+    handles.sort(Comparator.comparingInt(Map.Entry::getKey));
 
-    handles.sort(Comparator.comparing(Pair::getKey));
     List<Path> partHandles = handles
         .stream()
         .map(pair -> {
@@ -134,7 +133,10 @@ public class FileSystemMultipartUploader extends MultipartUploader {
         })
         .collect(Collectors.toList());
 
-    Path collectorPath = createCollectorPath(filePath);
+    byte[] uploadIdByteArray = multipartUploadId.toByteArray();
+    Path collectorPath = new Path(new String(uploadIdByteArray, 0,
+        uploadIdByteArray.length, Charsets.UTF_8));
+
     boolean emptyFile = totalPartsLen(partHandles) == 0;
     if (emptyFile) {
       fs.create(filePath).close();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
index b56dbf8..7ed987e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
@@ -17,40 +17,45 @@
  */
 package org.apache.hadoop.fs;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.List;
+import java.util.Map;
 
-import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang3.tuple.Pair;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 /**
  * MultipartUploader is an interface for copying files multipart and across
  * multiple nodes. Users should:
  * <ol>
- *   <li>Initialize an upload</li>
- *   <li>Upload parts in any order</li>
+ *   <li>Initialize an upload.</li>
+ *   <li>Upload parts in any order.</li>
  *   <li>Complete the upload in order to have it materialize in the destination
- *   FS</li>
+ *   FS.</li>
  * </ol>
- *
- * Implementers should make sure that the complete function should make sure
- * that 'complete' will reorder parts if the destination FS doesn't already
- * do it for them.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public abstract class MultipartUploader {
+public abstract class MultipartUploader implements Closeable {
   public static final Logger LOG =
       LoggerFactory.getLogger(MultipartUploader.class);
 
   /**
+   * Perform any cleanup.
+   * The upload is not required to support any operations after this.
+   * @throws IOException problems on close.
+   */
+  @Override
+  public void close() throws IOException {
+  }
+
+  /**
    * Initialize a multipart upload.
    * @param filePath Target path for upload.
    * @return unique identifier associating part uploads.
@@ -59,8 +64,8 @@ public abstract class MultipartUploader {
   public abstract UploadHandle initialize(Path filePath) throws IOException;
 
   /**
-   * Put part as part of a multipart upload. It should be possible to have
-   * parts uploaded in any order (or in parallel).
+   * Put part as part of a multipart upload.
+   * It is possible to have parts uploaded in any order (or in parallel).
    * @param filePath Target path for upload (same as {@link #initialize(Path)}).
    * @param inputStream Data for this part. Implementations MUST close this
    * stream after reading in the data.
@@ -77,15 +82,15 @@ public abstract class MultipartUploader {
   /**
    * Complete a multipart upload.
    * @param filePath Target path for upload (same as {@link #initialize(Path)}.
-   * @param handles non-empty list of identifiers with associated part numbers
+   * @param handles non-empty map of part number to part handle.
    *          from {@link #putPart(Path, InputStream, int, UploadHandle, long)}.
-   *          Depending on the backend, the list order may be significant.
    * @param multipartUploadId Identifier from {@link #initialize(Path)}.
    * @return unique PathHandle identifier for the uploaded file.
-   * @throws IOException IO failure or the handle list is empty.
+   * @throws IOException IO failure
    */
   public abstract PathHandle complete(Path filePath,
-      List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
+      Map<Integer, PartHandle> handles,
+      UploadHandle multipartUploadId)
       throws IOException;
 
   /**
@@ -98,13 +103,52 @@ public abstract class MultipartUploader {
       throws IOException;
 
   /**
-   * Utility method to validate uploadIDs
-   * @param uploadId
-   * @throws IllegalArgumentException
+   * Utility method to validate uploadIDs.
+   * @param uploadId Upload ID
+   * @throws IllegalArgumentException invalid ID
    */
   protected void checkUploadId(byte[] uploadId)
       throws IllegalArgumentException {
-    Preconditions.checkArgument(uploadId.length > 0,
+    checkArgument(uploadId != null, "null uploadId");
+    checkArgument(uploadId.length > 0,
         "Empty UploadId is not valid");
   }
+
+  /**
+   * Utility method to validate partHandles.
+   * @param partHandles handles
+   * @throws IllegalArgumentException if the parts are invalid
+   */
+  protected void checkPartHandles(Map<Integer, PartHandle> partHandles) {
+    checkArgument(!partHandles.isEmpty(),
+        "Empty upload");
+    partHandles.keySet()
+        .stream()
+        .forEach(key ->
+            checkArgument(key > 0,
+                "Invalid part handle index %s", key));
+  }
+
+  /**
+   * Check all the arguments to the
+   * {@link #putPart(Path, InputStream, int, UploadHandle, long)} operation.
+   * @param filePath Target path for upload (same as {@link #initialize(Path)}).
+   * @param inputStream Data for this part. Implementations MUST close this
+   * stream after reading in the data.
+   * @param partNumber Index of the part relative to others.
+   * @param uploadId Identifier from {@link #initialize(Path)}.
+   * @param lengthInBytes Target length to read from the stream.
+   * @throws IllegalArgumentException invalid argument
+   */
+  protected void checkPutArguments(Path filePath,
+      InputStream inputStream,
+      int partNumber,
+      UploadHandle uploadId,
+      long lengthInBytes) throws IllegalArgumentException {
+    checkArgument(filePath != null, "null filePath");
+    checkArgument(inputStream != null, "null inputStream");
+    checkArgument(partNumber > 0, "Invalid part number: %d", partNumber);
+    checkArgument(uploadId != null, "null uploadId");
+    checkArgument(lengthInBytes >= 0, "Invalid part length: %d", lengthInBytes);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
index 8b35232..e35b6bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
@@ -52,6 +52,13 @@ public abstract class MultipartUploaderFactory {
     }
   }
 
+  /**
+   * Get the multipart loader for a specific filesystem.
+   * @param fs filesystem
+   * @param conf configuration
+   * @return an uploader, or null if one was found.
+   * @throws IOException failure during the creation process.
+   */
   public static MultipartUploader get(FileSystem fs, Configuration conf)
       throws IOException {
     MultipartUploader mpu = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
index 532b6c7..6b4399e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
@@ -36,3 +36,4 @@ HDFS as these are commonly expected by Hadoop client applications.
 1. [FSDataOutputStreamBuilder class](fsdataoutputstreambuilder.html)
 2. [Testing with the Filesystem specification](testing.html)
 2. [Extending the specification and its tests](extending.html)
+1. [Uploading a file using Multiple Parts](multipartuploader.html)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
new file mode 100644
index 0000000..629c0c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
@@ -0,0 +1,235 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+
+<!--  ============================================================= -->
+<!--  CLASS: MultipartUploader -->
+<!--  ============================================================= -->
+
+# class `org.apache.hadoop.fs.MultipartUploader`
+
+<!-- MACRO{toc|fromDepth=1|toDepth=2} -->
+
+The abstract `MultipartUploader` class is the original class to upload a file
+using multiple parts to Hadoop-supported filesystems. The benefits of a
+multipart upload is that the file can be uploaded from multiple clients or
+processes in parallel and the results will not be visible to other clients until
+the `complete` function is called.
+
+When implemented by an object store, uploaded data may incur storage charges,
+even before it is visible in the filesystems. Users of this API must be diligent
+and always perform best-effort attempts to complete or abort the upload.
+
+## Invariants
+
+All the requirements of a valid MultipartUploader are considered implicit
+econditions and postconditions:
+all operations on a valid MultipartUploader MUST result in a new
+MultipartUploader that is also valid.
+
+The operations of a single multipart upload may take place across different
+instance of a multipart uploader, across different processes and hosts.
+It is therefore a requirement that:
+
+1. All state needed to upload a part, complete an upload or abort an upload
+must be contained within or retrievable from an upload handle.
+
+1. If an upload handle is marshalled to another process, then, if the
+receiving process has the correct permissions, it may participate in the
+upload, by uploading one or more parts, by completing an upload, and/or by
+aborting the upload.
+
+## Concurrency
+
+Multiple processes may upload parts of a multipart upload simultaneously.
+
+If a call is made to `initialize(path)` to a destination where an active
+upload is in progress, implementations MUST perform one of the two operations.
+
+* Reject the call as a duplicate.
+* Permit both to proceed, with the final output of the file being
+that of _exactly one of the two uploads_.
+
+Which upload succeeds is undefined. Users must not expect consistent
+behavior across filesystems, across filesystem instances *or even
+across different requests.
+
+If a multipart upload is completed or aborted while a part upload is in progress,
+the in-progress upload, if it has not completed, must not be included in
+the final file, in whole or in part. Implementations SHOULD raise an error
+in the `putPart()` operation.
+
+## Model
+
+A File System which supports Multipart Uploads extends the existing model
+`(Directories, Files, Symlinks)` to one of `(Directories, Files, Symlinks, Uploads)`
+`Uploads` of type `Map[UploadHandle -> Map[PartHandle -> UploadPart]`.
+
+
+The `Uploads` element of the state tuple is a map of all active uploads.
+
+```python
+Uploads: Map[UploadHandle -> Map[PartHandle -> UploadPart]`
+```
+
+An UploadHandle is a non-empty list of bytes.
+```python
+UploadHandle: List[byte]
+len(UploadHandle) > 0
+```
+
+Clients *MUST* treat this as opaque. What is core to this features design is that the handle is valid from
+across clients: the handle may be serialized on host `hostA`, deserialized on `hostB` and still used
+to extend or complete the upload.
+
+```python
+UploadPart = (Path: path, parts: Map[PartHandle -> byte[]])
+```
+
+Similarly, the `PartHandle` type is also a non-empty list of opaque bytes, again, marshallable between hosts.
+
+```python
+PartHandle: List[byte]
+```
+
+It is implicit that each `UploadHandle` in `FS.Uploads` is unique.
+Similarly, each `PartHandle` in the map of `[PartHandle -> UploadPart]` must also be unique.
+
+1. There is no requirement that Part Handles are unique across uploads.
+1. There is no requirement that Upload Handles are unique over time.
+However, if Part Handles are rapidly recycled, there is a risk that the nominally
+idempotent operation `abort(FS, uploadHandle)` could unintentionally cancel a
+successor operation which used the same Upload Handle.
+
+## State Changing Operations
+
+### `UploadHandle initialize(Path path)`
+
+Initialized a Multipart Upload, returning an upload handle for use in
+subsequent operations.
+
+#### Preconditions
+
+```python
+if path == "/" : raise IOException
+
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+```
+
+If a filesystem does not support concurrent uploads to a destination,
+then the following precondition is added
+
+```python
+if path in values(FS.Uploads) raise PathExistsException, IOException
+
+```
+
+
+#### Postconditions
+
+The outcome of this operation is that the filesystem state is updated with a new
+active upload, with a new handle, this handle being returned to the caller.
+
+```python
+handle' = UploadHandle where not handle' in keys(FS.Uploads)
+FS' = FS where FS'.Uploads(handle') == {}
+result = handle'
+```
+
+### `PartHandle putPart(Path path, InputStream inputStream, int partNumber, UploadHandle uploadHandle, long lengthInBytes)`
+
+Upload a part for the multipart upload.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads)
+partNumber >= 1
+lengthInBytes >= 0
+len(inputStream) >= lengthInBytes
+```
+
+#### Postconditions
+
+```python
+data' = inputStream(0..lengthInBytes)
+partHandle' = byte[] where not partHandle' in keys(FS.uploads(uploadHandle).parts)
+FS' = FS where FS'.uploads(uploadHandle).parts(partHandle') == data'
+result = partHandle'
+```
+
+The data is stored in the filesystem, pending completion.
+
+
+### `PathHandle complete(Path path, Map<Integer, PartHandle> parts, UploadHandle multipartUploadId)`
+
+Complete the multipart upload.
+
+A Filesystem may enforce a minimum size of each part, excluding the last part uploaded.
+
+If a part is out of this range, an `IOException` MUST be raised.
+
+#### Preconditions
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+FS.Uploads(uploadHandle).path == path
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+parts.size() > 0
+```
+
+If there are handles in the MPU which aren't included in the map, then the omitted
+parts will not be a part of the resulting file. It is up to the implementation
+of the MultipartUploader to make sure the leftover parts are cleaned up.
+
+In the case of backing stores that support directories (local filesystem, HDFS,
+etc), if, at the point of completion, there is now a directory at the
+destination then a `PathIsDirectoryException` or other `IOException` must be thrown.
+
+#### Postconditions
+
+```python
+UploadData' == ordered concatention of all data in the map of parts, ordered by key
+exists(FS', path') and result = PathHandle(path')
+FS' = FS where FS.Files(path) == UploadData' and not uploadHandle in keys(FS'.uploads)
+```
+
+The PathHandle is returned by the complete operation so subsequent operations
+will be able to identify that the data has not changed in the meantime.
+
+The order of parts in the uploaded by file is that of the natural order of
+parts: part 1 is ahead of part 2, etc.
+
+
+### `void abort(Path path, UploadHandle multipartUploadId)`
+
+Abort a multipart upload. The handle becomes invalid and not subject to reuse.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+```
+
+#### Postconditions
+
+The upload handle is no longer known.
+
+```python
+FS' = FS where not uploadHandle in keys(FS'.uploads)
+```
+A subsequent call to `abort()` with the same handle will fail, unless
+the handle has been recycled.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 7cee5a6..7a8f083 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -23,15 +23,19 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
 
 import com.google.common.base.Charsets;
+import org.junit.Assume;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BBUploadHandle;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -43,11 +47,64 @@ import org.apache.hadoop.fs.PathHandle;
 import org.apache.hadoop.fs.UploadHandle;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+import static org.apache.hadoop.test.LambdaTestUtils.eventually;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 public abstract class AbstractContractMultipartUploaderTest extends
     AbstractFSContractTestBase {
 
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
+  /**
+   * Size of very small uploads.
+   * Enough to be non empty, not big enough to cause delays on uploads.
+   */
+  protected static final int SMALL_FILE = 100;
+
+  private MultipartUploader mpu;
+  private MultipartUploader mpu2;
+  private final Random random = new Random();
+  private UploadHandle activeUpload;
+  private Path activeUploadPath;
+
+  protected String getMethodName() {
+    return methodName.getMethodName();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    Configuration conf = getContract().getConf();
+    mpu = MultipartUploaderFactory.get(getFileSystem(), conf);
+    mpu2 = MultipartUploaderFactory.get(getFileSystem(), conf);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    if (mpu!= null && activeUpload != null) {
+      try {
+        mpu.abort(activeUploadPath, activeUpload);
+      } catch (FileNotFoundException ignored) {
+        /* this is fine */
+      } catch (Exception e) {
+        LOG.info("in teardown", e);
+      }
+    }
+    cleanupWithLogger(LOG, mpu, mpu2);
+    super.teardown();
+  }
+
+  /**
+   * Get a test path based on the method name.
+   * @return a path to use in the test
+   * @throws IOException failure to build the path name up.
+   */
+  protected Path methodPath() throws IOException {
+    return path(getMethodName());
+  }
+
   /**
    * The payload is the part number repeated for the length of the part.
    * This makes checking the correctness of the upload straightforward.
@@ -55,9 +112,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
    * @return the bytes to upload.
    */
   private byte[] generatePayload(int partNumber) {
-    int sizeInBytes = partSizeInBytes();
-    ByteBuffer buffer = ByteBuffer.allocate(sizeInBytes);
-    for (int i=0 ; i < sizeInBytes/(Integer.SIZE/Byte.SIZE); ++i) {
+    return generatePayload(partNumber, partSizeInBytes());
+  }
+
+  /**
+   * Generate a payload of a given size; part number is used
+   * for all the values.
+   * @param partNumber part number
+   * @param size size in bytes
+   * @return the bytes to upload.
+   */
+  private byte[] generatePayload(final int partNumber, final int size) {
+    ByteBuffer buffer = ByteBuffer.allocate(size);
+    for (int i=0; i < size /(Integer.SIZE/Byte.SIZE); ++i) {
       buffer.putInt(partNumber);
     }
     return buffer.array();
@@ -70,11 +137,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
    * @throws IOException failure to read or digest the file.
    */
   protected byte[] digest(Path path) throws IOException {
-    FileSystem fs = getFileSystem();
-    try (InputStream in = fs.open(path)) {
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    try (InputStream in = getFileSystem().open(path)) {
       byte[] fdData = IOUtils.toByteArray(in);
       MessageDigest newDigest = DigestUtils.getMd5Digest();
-      return newDigest.digest(fdData);
+      byte[] digest = newDigest.digest(fdData);
+      return digest;
+    } finally {
+      timer.end("Download and digest of path %s", path);
     }
   }
 
@@ -93,74 +163,230 @@ public abstract class AbstractContractMultipartUploaderTest extends
   }
 
   /**
+   * How long in milliseconds for propagation of
+   * store changes, including update/delete/list
+   * to be everywhere.
+   * If 0: the FS is consistent.
+   * @return a time in milliseconds.
+   */
+  protected int timeToBecomeConsistentMillis() {
+    return 0;
+  }
+
+  /**
+   * Does a call to finalize an upload (either complete or abort) consume the
+   * uploadID immediately or is it reaped at a later point in time?
+   * @return true if the uploadID will be consumed immediately (and no longer
+   * resuable).
+   */
+  protected abstract boolean finalizeConsumesUploadIdImmediately();
+
+  /**
+   * Does the store support concurrent uploads to the same destination path?
+   * @return true if concurrent uploads are supported.
+   */
+  protected abstract boolean supportsConcurrentUploadsToSamePath();
+
+  /**
+   * Pick a multipart uploader from the index value.
+   * @param index index of upload
+   * @return an uploader
+   */
+  protected MultipartUploader mpu(int index) {
+    return (index % 2 == 0) ? mpu : mpu2;
+  }
+
+  /**
+   * Pick a multipart uploader at random.
+   * @return an uploader
+   */
+  protected MultipartUploader randomMpu() {
+    return mpu(random.nextInt(10));
+  }
+
+  /**
    * Assert that a multipart upload is successful.
    * @throws Exception failure
    */
   @Test
   public void testSingleUpload() throws Exception {
-    FileSystem fs = getFileSystem();
-    Path file = path("testSingleUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
     MessageDigest origDigest = DigestUtils.getMd5Digest();
-    byte[] payload = generatePayload(1);
+    int size = SMALL_FILE;
+    byte[] payload = generatePayload(1, size);
     origDigest.update(payload);
-    InputStream is = new ByteArrayInputStream(payload);
-    PartHandle partHandle = mpu.putPart(file, is, 1, uploadHandle,
-        payload.length);
-    partHandles.add(Pair.of(1, partHandle));
-    PathHandle fd = completeUpload(file, mpu, uploadHandle, partHandles,
+    PartHandle partHandle = putPart(file, uploadHandle, 1, payload);
+    partHandles.put(1, partHandle);
+    PathHandle fd = completeUpload(file, uploadHandle, partHandles,
         origDigest,
-        payload.length);
+        size);
+
+    if (finalizeConsumesUploadIdImmediately()) {
+      intercept(FileNotFoundException.class,
+          () -> mpu.complete(file, partHandles, uploadHandle));
+    } else {
+      PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
+      assertArrayEquals("Path handles differ", fd.toByteArray(),
+          fd2.toByteArray());
+    }
+  }
+
+  /**
+   * Initialize an upload.
+   * This saves the path and upload handle as the active
+   * upload, for aborting in teardown
+   * @param dest destination
+   * @return the handle
+   * @throws IOException failure to initialize
+   */
+  protected UploadHandle initializeUpload(final Path dest) throws IOException {
+    activeUploadPath = dest;
+    activeUpload = randomMpu().initialize(dest);
+    return activeUpload;
+  }
 
-    // Complete is idempotent
-    PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
-    assertArrayEquals("Path handles differ", fd.toByteArray(),
-        fd2.toByteArray());
+  /**
+   * Generate then upload a part.
+   * @param file destination
+   * @param uploadHandle handle
+   * @param index index of part
+   * @param origDigest digest to build up. May be null
+   * @return the part handle
+   * @throws IOException IO failure.
+   */
+  protected PartHandle buildAndPutPart(
+      final Path file,
+      final UploadHandle uploadHandle,
+      final int index,
+      final MessageDigest origDigest) throws IOException {
+    byte[] payload = generatePayload(index);
+    if (origDigest != null) {
+      origDigest.update(payload);
+    }
+    return putPart(file, uploadHandle, index, payload);
   }
 
+  /**
+   * Put a part.
+   * The entire byte array is uploaded.
+   * @param file destination
+   * @param uploadHandle handle
+   * @param index index of part
+   * @param payload byte array of payload
+   * @return the part handle
+   * @throws IOException IO failure.
+   */
+  protected PartHandle putPart(final Path file,
+      final UploadHandle uploadHandle,
+      final int index,
+      final byte[] payload) throws IOException {
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    PartHandle partHandle = mpu(index)
+        .putPart(file,
+            new ByteArrayInputStream(payload),
+            index,
+            uploadHandle,
+            payload.length);
+    timer.end("Uploaded part %s", index);
+    LOG.info("Upload bandwidth {} MB/s",
+        timer.bandwidthDescription(payload.length));
+    return partHandle;
+  }
+
+  /**
+   * Complete an upload with the active MPU instance.
+   * @param file destination
+   * @param uploadHandle handle
+   * @param partHandles map of handles
+   * @param origDigest digest of source data (may be null)
+   * @param expectedLength expected length of result.
+   * @return the path handle from the upload.
+   * @throws IOException IO failure
+   */
   private PathHandle completeUpload(final Path file,
-      final MultipartUploader mpu,
       final UploadHandle uploadHandle,
-      final List<Pair<Integer, PartHandle>> partHandles,
+      final Map<Integer, PartHandle> partHandles,
       final MessageDigest origDigest,
       final int expectedLength) throws IOException {
-    PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+    PathHandle fd = complete(file, uploadHandle, partHandles);
 
     FileStatus status = verifyPathExists(getFileSystem(),
         "Completed file", file);
     assertEquals("length of " + status,
         expectedLength, status.getLen());
 
+    if (origDigest != null) {
+      verifyContents(file, origDigest, expectedLength);
+    }
+    return fd;
+  }
+
+  /**
+   * Verify the contents of a file.
+   * @param file path
+   * @param origDigest digest
+   * @param expectedLength expected length (for logging B/W)
+   * @throws IOException IO failure
+   */
+  protected void verifyContents(final Path file,
+                                final MessageDigest origDigest,
+                                final int expectedLength) throws IOException {
+    ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
     assertArrayEquals("digest of source and " + file
             + " differ",
         origDigest.digest(), digest(file));
+    timer2.end("Completed digest", file);
+    LOG.info("Download bandwidth {} MB/s",
+        timer2.bandwidthDescription(expectedLength));
+  }
+
+  /**
+   * Perform the inner complete without verification.
+   * @param file destination path
+   * @param uploadHandle upload handle
+   * @param partHandles map of parts
+   * @return the path handle from the upload.
+   * @throws IOException IO failure
+   */
+  private PathHandle complete(final Path file,
+      final UploadHandle uploadHandle,
+      final Map<Integer, PartHandle> partHandles) throws IOException {
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    PathHandle fd = randomMpu().complete(file, partHandles, uploadHandle);
+    timer.end("Completed upload to %s", file);
     return fd;
   }
 
   /**
+   * Abort an upload.
+   * @param file path
+   * @param uploadHandle handle
+   * @throws IOException failure
+   */
+  private void abortUpload(final Path file, UploadHandle uploadHandle)
+      throws IOException {
+    randomMpu().abort(file, uploadHandle);
+  }
+
+  /**
    * Assert that a multipart upload is successful.
    * @throws Exception failure
    */
   @Test
   public void testMultipartUpload() throws Exception {
-    FileSystem fs = getFileSystem();
-    Path file = path("testMultipartUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
     MessageDigest origDigest = DigestUtils.getMd5Digest();
     final int payloadCount = getTestPayloadCount();
     for (int i = 1; i <= payloadCount; ++i) {
-      byte[] payload = generatePayload(i);
-      origDigest.update(payload);
-      InputStream is = new ByteArrayInputStream(payload);
-      PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
-          payload.length);
-      partHandles.add(Pair.of(i, partHandle));
+      PartHandle partHandle = buildAndPutPart(file, uploadHandle, i,
+          origDigest);
+      partHandles.put(i, partHandle);
     }
-    completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+    completeUpload(file, uploadHandle, partHandles, origDigest,
         payloadCount * partSizeInBytes());
   }
 
@@ -173,17 +399,33 @@ public abstract class AbstractContractMultipartUploaderTest extends
   public void testMultipartUploadEmptyPart() throws Exception {
     FileSystem fs = getFileSystem();
     Path file = path("testMultipartUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
-    MessageDigest origDigest = DigestUtils.getMd5Digest();
-    byte[] payload = new byte[0];
-    origDigest.update(payload);
-    InputStream is = new ByteArrayInputStream(payload);
-    PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
-        payload.length);
-      partHandles.add(Pair.of(0, partHandle));
-    completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+    try (MultipartUploader uploader =
+             MultipartUploaderFactory.get(fs, null)) {
+      UploadHandle uploadHandle = uploader.initialize(file);
+
+      Map<Integer, PartHandle> partHandles = new HashMap<>();
+      MessageDigest origDigest = DigestUtils.getMd5Digest();
+      byte[] payload = new byte[0];
+      origDigest.update(payload);
+      InputStream is = new ByteArrayInputStream(payload);
+      PartHandle partHandle = uploader.putPart(file, is, 1, uploadHandle,
+          payload.length);
+      partHandles.put(1, partHandle);
+      completeUpload(file, uploadHandle, partHandles, origDigest, 0);
+    }
+  }
+
+  /**
+   * Assert that a multipart upload is successful.
+   * @throws Exception failure
+   */
+  @Test
+  public void testUploadEmptyBlock() throws Exception {
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
+    partHandles.put(1, putPart(file, uploadHandle, 1, new byte[0]));
+    completeUpload(file, uploadHandle, partHandles, null, 0);
   }
 
   /**
@@ -192,11 +434,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
    */
   @Test
   public void testMultipartUploadReverseOrder() throws Exception {
-    FileSystem fs = getFileSystem();
-    Path file = path("testMultipartUploadReverseOrder");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
     MessageDigest origDigest = DigestUtils.getMd5Digest();
     final int payloadCount = getTestPayloadCount();
     for (int i = 1; i <= payloadCount; ++i) {
@@ -204,13 +444,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
       origDigest.update(payload);
     }
     for (int i = payloadCount; i > 0; --i) {
-      byte[] payload = generatePayload(i);
-      InputStream is = new ByteArrayInputStream(payload);
-      PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
-          payload.length);
-      partHandles.add(Pair.of(i, partHandle));
+      partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
     }
-    completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+    completeUpload(file, uploadHandle, partHandles, origDigest,
         payloadCount * partSizeInBytes());
   }
 
@@ -222,25 +458,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
   public void testMultipartUploadReverseOrderNonContiguousPartNumbers()
       throws Exception {
     describe("Upload in reverse order and the part numbers are not contiguous");
-    FileSystem fs = getFileSystem();
-    Path file = path("testMultipartUploadReverseOrderNonContiguousPartNumbers");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
     MessageDigest origDigest = DigestUtils.getMd5Digest();
     int payloadCount = 2 * getTestPayloadCount();
     for (int i = 2; i <= payloadCount; i += 2) {
       byte[] payload = generatePayload(i);
       origDigest.update(payload);
     }
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
     for (int i = payloadCount; i > 0; i -= 2) {
-      byte[] payload = generatePayload(i);
-      InputStream is = new ByteArrayInputStream(payload);
-      PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
-          payload.length);
-      partHandles.add(Pair.of(i, partHandle));
+      partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
     }
-    completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+    completeUpload(file, uploadHandle, partHandles, origDigest,
         getTestPayloadCount() * partSizeInBytes());
   }
 
@@ -251,19 +481,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
   @Test
   public void testMultipartUploadAbort() throws Exception {
     describe("Upload and then abort it before completing");
-    FileSystem fs = getFileSystem();
-    Path file = path("testMultipartUploadAbort");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle uploadHandle = mpu.initialize(file);
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
-    for (int i = 20; i >= 10; --i) {
-      byte[] payload = generatePayload(i);
-      InputStream is = new ByteArrayInputStream(payload);
-      PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
-          payload.length);
-      partHandles.add(Pair.of(i, partHandle));
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    int end = 10;
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
+    for (int i = 12; i > 10; i--) {
+      partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
     }
-    mpu.abort(file, uploadHandle);
+    abortUpload(file, uploadHandle);
 
     String contents = "ThisIsPart49\n";
     int len = contents.getBytes(Charsets.UTF_8).length;
@@ -275,6 +500,15 @@ public abstract class AbstractContractMultipartUploaderTest extends
         () -> mpu.complete(file, partHandles, uploadHandle));
 
     assertPathDoesNotExist("Uploaded file should not exist", file);
+
+    // A second abort should be an FileNotFoundException if the UploadHandle is
+    // consumed by finalization operations (complete, abort).
+    if (finalizeConsumesUploadIdImmediately()) {
+      intercept(FileNotFoundException.class,
+          () -> abortUpload(file, uploadHandle));
+    } else {
+      abortUpload(file, uploadHandle);
+    }
   }
 
   /**
@@ -282,13 +516,23 @@ public abstract class AbstractContractMultipartUploaderTest extends
    */
   @Test
   public void testAbortUnknownUpload() throws Exception {
-    FileSystem fs = getFileSystem();
-    Path file = path("testAbortUnknownUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+    Path file = methodPath();
     ByteBuffer byteBuffer = ByteBuffer.wrap(
         "invalid-handle".getBytes(Charsets.UTF_8));
     UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
-    intercept(FileNotFoundException.class, () -> mpu.abort(file, uploadHandle));
+    intercept(FileNotFoundException.class,
+        () -> abortUpload(file, uploadHandle));
+  }
+
+  /**
+   * Trying to abort with a handle of size 0 must fail.
+   */
+  @Test
+  public void testAbortEmptyUpload() throws Exception {
+    describe("initialize upload and abort before uploading data");
+    Path file = methodPath();
+    abortUpload(file, initializeUpload(file));
+    assertPathDoesNotExist("Uploaded file should not exist", file);
   }
 
   /**
@@ -296,13 +540,10 @@ public abstract class AbstractContractMultipartUploaderTest extends
    */
   @Test
   public void testAbortEmptyUploadHandle() throws Exception {
-    FileSystem fs = getFileSystem();
-    Path file = path("testAbortEmptyUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
     ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[0]);
     UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
     intercept(IllegalArgumentException.class,
-        () -> mpu.abort(file, uploadHandle));
+        () -> abortUpload(methodPath(), uploadHandle));
   }
 
   /**
@@ -311,26 +552,20 @@ public abstract class AbstractContractMultipartUploaderTest extends
   @Test
   public void testCompleteEmptyUpload() throws Exception {
     describe("Expect an empty MPU to fail, but still be abortable");
-    FileSystem fs = getFileSystem();
-    Path dest = path("testCompleteEmptyUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle handle = mpu.initialize(dest);
-    intercept(IOException.class,
-        () -> mpu.complete(dest, new ArrayList<>(), handle));
-    mpu.abort(dest, handle);
+    Path dest = methodPath();
+    UploadHandle handle = initializeUpload(dest);
+    intercept(IllegalArgumentException.class,
+        () -> mpu.complete(dest, new HashMap<>(), handle));
+    abortUpload(dest, handle);
   }
 
   /**
    * When we pass empty uploadID, putPart throws IllegalArgumentException.
-   * @throws Exception
    */
   @Test
   public void testPutPartEmptyUploadID() throws Exception {
     describe("Expect IllegalArgumentException when putPart uploadID is empty");
-    FileSystem fs = getFileSystem();
-    Path dest = path("testCompleteEmptyUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    mpu.initialize(dest);
+    Path dest = methodPath();
     UploadHandle emptyHandle =
         BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
     byte[] payload = generatePayload(1);
@@ -341,25 +576,123 @@ public abstract class AbstractContractMultipartUploaderTest extends
 
   /**
    * When we pass empty uploadID, complete throws IllegalArgumentException.
-   * @throws Exception
    */
   @Test
   public void testCompleteEmptyUploadID() throws Exception {
     describe("Expect IllegalArgumentException when complete uploadID is empty");
-    FileSystem fs = getFileSystem();
-    Path dest = path("testCompleteEmptyUpload");
-    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
-    UploadHandle realHandle = mpu.initialize(dest);
+    Path dest = methodPath();
+    UploadHandle realHandle = initializeUpload(dest);
     UploadHandle emptyHandle =
         BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
-    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
-    byte[] payload = generatePayload(1);
-    InputStream is = new ByteArrayInputStream(payload);
-    PartHandle partHandle = mpu.putPart(dest, is, 1, realHandle,
-        payload.length);
-    partHandles.add(Pair.of(1, partHandle));
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
+    PartHandle partHandle = putPart(dest, realHandle, 1,
+        generatePayload(1, SMALL_FILE));
+    partHandles.put(1, partHandle);
 
     intercept(IllegalArgumentException.class,
         () -> mpu.complete(dest, partHandles, emptyHandle));
+
+    // and, while things are setup, attempt to complete with
+    // a part index of 0
+    partHandles.clear();
+    partHandles.put(0, partHandle);
+    intercept(IllegalArgumentException.class,
+        () -> mpu.complete(dest, partHandles, realHandle));
+  }
+
+  /**
+   * Assert that upon completion, a directory in the way of the file will
+   * result in a failure. This test only applies to backing stores with a
+   * concept of directories.
+   * @throws Exception failure
+   */
+  @Test
+  public void testDirectoryInTheWay() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path file = methodPath();
+    UploadHandle uploadHandle = initializeUpload(file);
+    Map<Integer, PartHandle> partHandles = new HashMap<>();
+    int size = SMALL_FILE;
+    PartHandle partHandle = putPart(file, uploadHandle, 1,
+        generatePayload(1, size));
+    partHandles.put(1, partHandle);
+
+    fs.mkdirs(file);
+    intercept(IOException.class,
+        () -> completeUpload(file, uploadHandle, partHandles, null,
+            size));
+    // abort should still work
+    abortUpload(file, uploadHandle);
+  }
+
+  @Test
+  public void testConcurrentUploads() throws Throwable {
+
+    // if the FS doesn't support concurrent uploads, this test is
+    // required to fail during the second initialization.
+    final boolean concurrent = supportsConcurrentUploadsToSamePath();
+
+    describe("testing concurrent uploads, MPU support for this is "
+        + concurrent);
+    final FileSystem fs = getFileSystem();
+    final Path file = methodPath();
+    final int size1 = SMALL_FILE;
+    final int partId1 = 1;
+    final byte[] payload1 = generatePayload(partId1, size1);
+    final MessageDigest digest1 = DigestUtils.getMd5Digest();
+    digest1.update(payload1);
+    final UploadHandle upload1 = initializeUpload(file);
+    final Map<Integer, PartHandle> partHandles1 = new HashMap<>();
+
+    // initiate part 2
+    // by using a different size, it's straightforward to see which
+    // version is visible, before reading/digesting the contents
+    final int size2 = size1 * 2;
+    final int partId2 = 2;
+    final byte[] payload2 = generatePayload(partId1, size2);
+    final MessageDigest digest2 = DigestUtils.getMd5Digest();
+    digest2.update(payload2);
+
+    final UploadHandle upload2;
+    try {
+      upload2 = initializeUpload(file);
+      Assume.assumeTrue(
+          "The Filesystem is unexpectedly supporting concurrent uploads",
+          concurrent);
+    } catch (IOException e) {
+      if (!concurrent) {
+        // this is expected, so end the test
+        LOG.debug("Expected exception raised on concurrent uploads {}", e);
+        return;
+      } else {
+        throw e;
+      }
+    }
+    final Map<Integer, PartHandle> partHandles2 = new HashMap<>();
+
+
+    assertNotEquals("Upload handles match", upload1, upload2);
+
+    // put part 1
+    partHandles1.put(partId1, putPart(file, upload1, partId1, payload1));
+
+    // put part2
+    partHandles2.put(partId2, putPart(file, upload2, partId2, payload2));
+
+    // complete part u1. expect its size and digest to
+    // be as expected.
+    completeUpload(file, upload1, partHandles1, digest1, size1);
+
+    // now upload part 2.
+    complete(file, upload2, partHandles2);
+    // and await the visible length to match
+    eventually(timeToBecomeConsistentMillis(), 500,
+        () -> {
+          FileStatus status = fs.getFileStatus(file);
+          assertEquals("File length in " + status,
+              size2, status.getLen());
+        });
+
+    verifyContents(file, digest2, size2);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
index a50d2e4..6e27964d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
@@ -40,4 +40,14 @@ public class TestLocalFSContractMultipartUploader
   protected int partSizeInBytes() {
     return 1024;
   }
+
+  @Override
+  protected boolean finalizeConsumesUploadIdImmediately() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsConcurrentUploadsToSamePath() {
+    return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
index f3a5265..54f4ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
@@ -21,6 +21,8 @@ import java.io.IOException;
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
@@ -32,6 +34,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
 public class TestHDFSContractMultipartUploader extends
     AbstractContractMultipartUploaderTest {
 
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
   @BeforeClass
   public static void createCluster() throws IOException {
     HDFSContract.createCluster();
@@ -55,4 +60,14 @@ public class TestHDFSContractMultipartUploader extends
   protected int partSizeInBytes() {
     return 1024;
   }
+
+  @Override
+  protected boolean finalizeConsumesUploadIdImmediately() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsConcurrentUploadsToSamePath() {
+    return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
index cab4e2a..cf58751 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
@@ -25,7 +25,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
@@ -68,10 +70,8 @@ public class S3AMultipartUploader extends MultipartUploader {
   public static final String HEADER = "S3A-part01";
 
   public S3AMultipartUploader(FileSystem fs, Configuration conf) {
-    if (!(fs instanceof S3AFileSystem)) {
-      throw new IllegalArgumentException(
-          "S3A MultipartUploads must use S3AFileSystem");
-    }
+    Preconditions.checkArgument(fs instanceof S3AFileSystem,
+        "Wrong filesystem: expected S3A but got %s", fs);
     s3a = (S3AFileSystem) fs;
   }
 
@@ -88,6 +88,8 @@ public class S3AMultipartUploader extends MultipartUploader {
   public PartHandle putPart(Path filePath, InputStream inputStream,
       int partNumber, UploadHandle uploadId, long lengthInBytes)
       throws IOException {
+    checkPutArguments(filePath, inputStream, partNumber, uploadId,
+        lengthInBytes);
     byte[] uploadIdBytes = uploadId.toByteArray();
     checkUploadId(uploadIdBytes);
     String key = s3a.pathToKey(filePath);
@@ -105,14 +107,16 @@ public class S3AMultipartUploader extends MultipartUploader {
 
   @Override
   public PathHandle complete(Path filePath,
-      List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId)
+      Map<Integer, PartHandle> handleMap,
+      UploadHandle uploadId)
       throws IOException {
     byte[] uploadIdBytes = uploadId.toByteArray();
     checkUploadId(uploadIdBytes);
-    if (handles.isEmpty()) {
-      throw new IOException("Empty upload");
-    }
 
+    checkPartHandles(handleMap);
+    List<Map.Entry<Integer, PartHandle>> handles =
+        new ArrayList<>(handleMap.entrySet());
+    handles.sort(Comparator.comparingInt(Map.Entry::getKey));
     final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper();
     String key = s3a.pathToKey(filePath);
 
@@ -121,11 +125,11 @@ public class S3AMultipartUploader extends MultipartUploader {
     ArrayList<PartETag> eTags = new ArrayList<>();
     eTags.ensureCapacity(handles.size());
     long totalLength = 0;
-    for (Pair<Integer, PartHandle> handle : handles) {
-      byte[] payload = handle.getRight().toByteArray();
+    for (Map.Entry<Integer, PartHandle> handle : handles) {
+      byte[] payload = handle.getValue().toByteArray();
       Pair<Long, String> result = parsePartHandlePayload(payload);
       totalLength += result.getLeft();
-      eTags.add(new PartETag(handle.getLeft(), result.getRight()));
+      eTags.add(new PartETag(handle.getKey(), result.getRight()));
     }
     AtomicInteger errorCount = new AtomicInteger(0);
     CompleteMultipartUploadResult result = writeHelper.completeMPUwithRetries(
@@ -172,7 +176,7 @@ public class S3AMultipartUploader extends MultipartUploader {
       throws IOException {
     Preconditions.checkArgument(StringUtils.isNotEmpty(eTag),
         "Empty etag");
-    Preconditions.checkArgument(len > 0,
+    Preconditions.checkArgument(len >= 0,
         "Invalid length");
 
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -190,6 +194,7 @@ public class S3AMultipartUploader extends MultipartUploader {
    * @return the length and etag
    * @throws IOException error reading the payload
    */
+  @VisibleForTesting
   static Pair<Long, String> parsePartHandlePayload(byte[] data)
       throws IOException {
 
@@ -201,7 +206,7 @@ public class S3AMultipartUploader extends MultipartUploader {
       }
       final long len = input.readLong();
       final String etag = input.readUTF();
-      if (len <= 0) {
+      if (len < 0) {
         throw new IOException("Negative length");
       }
       return Pair.of(len, etag);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
index d28f39b..059312a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.fs.contract.s3a;
 
-import java.io.IOException;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -26,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 
@@ -35,6 +34,9 @@ import static org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.DEFAULT_H
 
 /**
  * Test MultipartUploader with S3A.
+ * Although not an S3A Scale test subclass, it uses the -Dscale option
+ * to enable it, and partition size option to control the size of
+ * parts uploaded.
  */
 public class ITestS3AContractMultipartUploader extends
     AbstractContractMultipartUploaderTest {
@@ -79,6 +81,35 @@ public class ITestS3AContractMultipartUploader extends
     return new S3AContract(conf);
   }
 
+  /**
+   * Bigger test: use the scale timeout.
+   * @return the timeout for scale tests.
+   */
+  @Override
+  protected int getTestTimeoutMillis() {
+    return SCALE_TEST_TIMEOUT_MILLIS;
+  }
+
+
+  @Override
+  protected boolean supportsConcurrentUploadsToSamePath() {
+    return true;
+  }
+
+  /**
+   * Provide a pessimistic time to become consistent.
+   * @return a time in milliseconds
+   */
+  @Override
+  protected int timeToBecomeConsistentMillis() {
+    return 30 * 1000;
+  }
+
+  @Override
+  protected boolean finalizeConsumesUploadIdImmediately() {
+    return false;
+  }
+
   @Override
   public void setup() throws Exception {
     super.setup();
@@ -103,14 +134,29 @@ public class ITestS3AContractMultipartUploader extends
   public void teardown() throws Exception {
     Path teardown = path("teardown").getParent();
     S3AFileSystem fs = getFileSystem();
-    WriteOperationHelper helper = fs.getWriteOperationHelper();
-    try {
-      LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
-      int count = helper.abortMultipartUploadsUnderPath(fs.pathToKey(teardown));
-      LOG.info("Found {} incomplete uploads", count);
-    } catch (IOException e) {
-      LOG.warn("IOE in teardown", e);
+    if (fs != null) {
+      WriteOperationHelper helper = fs.getWriteOperationHelper();
+      try {
+        LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
+        int count = helper.abortMultipartUploadsUnderPath(
+            fs.pathToKey(teardown));
+        LOG.info("Found {} incomplete uploads", count);
+      } catch (Exception e) {
+        LOG.warn("Exeception in teardown", e);
+      }
     }
     super.teardown();
   }
+
+  /**
+   * S3 has no concept of directories, so this test does not apply.
+   */
+  public void testDirectoryInTheWay() throws Exception {
+    // no-op
+  }
+
+  @Override
+  public void testMultipartUploadReverseOrder() throws Exception {
+    ContractTestUtils.skip("skipped for speed");
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
index 35d0460..4825d26 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
@@ -60,7 +60,7 @@ public class TestS3AMultipartUploaderSupport extends HadoopTestBase {
   @Test
   public void testNoLen() throws Throwable {
     intercept(IllegalArgumentException.class,
-        () -> buildPartHandlePayload("tag", 0));
+        () -> buildPartHandlePayload("tag", -1));
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[43/50] [abbrv] hadoop git commit: HDDS-836. Add TokenIdentifier Ozone for delegation token and block token. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-836. Add TokenIdentifier Ozone for delegation token and block token. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c0baf0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c0baf0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c0baf0e

Branch: refs/heads/HDDS-4
Commit: 9c0baf0ea30a0dc38778c6030eca1468ed7e40ae
Parents: 4770e9d
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 14 14:26:33 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../hdds/security/x509/keys/SecurityUtil.java   |  59 ++++
 hadoop-hdds/common/src/main/proto/hdds.proto    |  24 ++
 .../security/OzoneBlockTokenIdentifier.java     | 178 +++++++++++
 .../ozone/security/OzoneBlockTokenSelector.java |  55 ++++
 .../security/OzoneDelegationTokenSelector.java  |  52 ++++
 .../hadoop/ozone/security/OzoneSecretKey.java   | 195 ++++++++++++
 .../ozone/security/OzoneTokenIdentifier.java    | 217 ++++++++++++++
 .../hadoop/ozone/security/package-info.java     |  21 ++
 .../src/main/proto/OzoneManagerProtocol.proto   |  20 ++
 .../security/TestOzoneBlockTokenIdentifier.java | 255 ++++++++++++++++
 .../security/TestOzoneTokenIdentifier.java      | 300 +++++++++++++++++++
 .../hadoop/ozone/security/package-info.java     |  21 ++
 12 files changed, 1397 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
index 2ca8825..6147d3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -18,6 +18,15 @@
  */
 package org.apache.hadoop.hdds.security.x509.keys;
 
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
 import org.bouncycastle.asn1.ASN1Sequence;
@@ -76,4 +85,54 @@ public final class SecurityUtil {
     }
     throw new CertificateException("No PKCS#9 extension found in CSR");
   }
+
+  /*
+   * Returns private key created from encoded key.
+   * @return private key if successful else returns null.
+   */
+  public static PrivateKey getPrivateKey(byte[] encodedKey,
+      SecurityConfig secureConfig) {
+    PrivateKey pvtKey = null;
+    if (encodedKey == null || encodedKey.length == 0) {
+      return null;
+    }
+
+    try {
+      KeyFactory kf = null;
+
+      kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+          secureConfig.getProvider());
+      pvtKey = kf.generatePrivate(new PKCS8EncodedKeySpec(encodedKey));
+
+    } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+        NoSuchProviderException e) {
+      return null;
+    }
+    return pvtKey;
+  }
+
+  /*
+   * Returns public key created from encoded key.
+   * @return public key if successful else returns null.
+   */
+  public static PublicKey getPublicKey(byte[] encodedKey,
+      SecurityConfig secureConfig) {
+    PublicKey key = null;
+    if (encodedKey == null || encodedKey.length == 0) {
+      return null;
+    }
+
+    try {
+      KeyFactory kf = null;
+      kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+          secureConfig.getProvider());
+      key = kf.generatePublic(new X509EncodedKeySpec(encodedKey));
+
+    } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+        NoSuchProviderException e) {
+      return null;
+    }
+    return key;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index cf3d6d4..8598fbf 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -189,6 +189,30 @@ message ContainerBlockID {
     required int64 localID = 2;
 }
 
+
+/**
+ * Information for the Hdds block token.
+ * When adding further fields, make sure they are optional as they would
+ * otherwise not be backwards compatible.
+ */
+message BlockTokenSecretProto {
+    /**
+ * File access permissions mode.
+ */
+    enum AccessModeProto {
+        READ = 1;
+        WRITE = 2;
+        COPY = 3;
+        DELETE = 4;
+    }
+    required string ownerId = 1;
+    required string blockId = 2;
+    required uint64 expiryDate = 3;
+    required string omCertSerialId = 4;
+    repeated AccessModeProto modes = 5;
+
+}
+
 message BlockID {
     required ContainerBlockID containerBlockID = 1;
     optional uint64 blockCommitSequenceId = 2 [default = 0];

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..92b6d16
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+  static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+  private long expiryDate;
+  private String ownerId;
+  private String blockId;
+  private final EnumSet<AccessModeProto> modes;
+  private final String omCertSerialId;
+
+  public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+      EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
+    this.ownerId = ownerId;
+    this.blockId = blockId;
+    this.expiryDate = expiryDate;
+    this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+    this.omCertSerialId = omCertSerialId;
+  }
+
+  @Override
+  public UserGroupInformation getUser() {
+    if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+      return UserGroupInformation.createRemoteUser(blockId);
+    }
+    return UserGroupInformation.createRemoteUser(ownerId);
+  }
+
+  public long getExpiryDate() {
+    return expiryDate;
+  }
+
+  public String getOwnerId() {
+    return ownerId;
+  }
+
+  public String getBlockId() {
+    return blockId;
+  }
+
+  public EnumSet<AccessModeProto> getAccessModes() {
+    return modes;
+  }
+
+  public String getOmCertSerialId(){
+    return omCertSerialId;
+  }
+
+  @Override
+  public Text getKind() {
+    return KIND_NAME;
+  }
+
+  @Override
+  public String toString() {
+    return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+        + ", ownerId=" + this.getOwnerId()
+        + ", omCertSerialId=" + this.getOmCertSerialId()
+        + ", blockId=" + this.getBlockId() + ", access modes="
+        + this.getAccessModes() + ")";
+  }
+
+  static boolean isEqual(Object a, Object b) {
+    return a == null ? b == null : a.equals(b);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+
+    if (obj instanceof OzoneBlockTokenIdentifier) {
+      OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+      return new EqualsBuilder()
+          .append(this.expiryDate, that.expiryDate)
+          .append(this.ownerId, that.ownerId)
+          .append(this.blockId, that.blockId)
+          .append(this.modes, that.modes)
+          .append(this.omCertSerialId, that.omCertSerialId)
+          .build();
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(133, 567)
+        .append(this.expiryDate)
+        .append(this.blockId)
+        .append(this.ownerId)
+        .append(this.modes)
+        .append(this.omCertSerialId)
+        .build();
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    final DataInputStream dis = (DataInputStream) in;
+    if (!dis.markSupported()) {
+      throw new IOException("Could not peek first byte.");
+    }
+    readFieldsProtobuf(dis);
+  }
+
+  @VisibleForTesting
+  public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+      throws IOException {
+    BlockTokenSecretProto tokenPtoto =
+        BlockTokenSecretProto.parseFrom((DataInputStream) in);
+    return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+        tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+        tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    writeProtobuf(out);
+  }
+
+  @VisibleForTesting
+  void writeProtobuf(DataOutput out) throws IOException {
+    Builder builder = BlockTokenSecretProto.newBuilder()
+        .setBlockId(this.getBlockId())
+        .setOwnerId(this.getOwnerId())
+        .setOmCertSerialId(this.getOmCertSerialId())
+        .setExpiryDate(this.getExpiryDate());
+    // Add access mode allowed
+    for (AccessModeProto mode : this.getAccessModes()) {
+      builder.addModes(AccessModeProto.valueOf(mode.name()));
+    }
+    out.write(builder.build().toByteArray());
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..086e19b
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+    TokenSelector<OzoneBlockTokenIdentifier> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneBlockTokenSelector.class);
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+      Collection<Token<? extends TokenIdentifier>> tokens) {
+    if (service == null) {
+      return null;
+    }
+    for (Token<? extends TokenIdentifier> token : tokens) {
+      if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+        LOG.trace("Getting token for service:{}", service);
+        return (Token<OzoneBlockTokenIdentifier>) token;
+      }
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
new file mode 100644
index 0000000..0d63c4e
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.util.Collection;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A delegation token selector that is specialized for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneDelegationTokenSelector
+    extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+
+  public OzoneDelegationTokenSelector() {
+    super(OzoneTokenIdentifier.KIND_NAME);
+  }
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(OzoneDelegationTokenSelector.class);
+
+  @Override
+  public Token selectToken(Text service,
+      Collection<Token<? extends TokenIdentifier>> tokens) {
+    LOG.trace("Getting token for service {}", service);
+    return super.selectToken(service, tokens);
+  }
+
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
new file mode 100644
index 0000000..9b2f912
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SecretKeyProto;
+
+/**
+ * Wrapper class for Ozone/Hdds secret keys. Used in delegation tokens and block
+ * tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretKey implements Writable {
+
+  private int keyId;
+  private long expiryDate;
+  private PrivateKey privateKey;
+  private PublicKey publicKey;
+  private int maxKeyLen;
+  private SecurityConfig securityConfig;
+
+  public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair,
+      int maxKeyLen) {
+    Preconditions.checkNotNull(keyId);
+    this.keyId = keyId;
+    this.expiryDate = expiryDate;
+    byte[] encodedKey = keyPair.getPrivate().getEncoded();
+    this.maxKeyLen = maxKeyLen;
+    if (encodedKey.length > maxKeyLen) {
+      throw new RuntimeException("can't create " + encodedKey.length +
+          " byte long DelegationKey.");
+    }
+    this.privateKey = keyPair.getPrivate();
+    this.publicKey = keyPair.getPublic();
+  }
+
+  /*
+   * Create new instance using default signature algorithm and provider.
+   * */
+  public OzoneSecretKey(int keyId, long expiryDate, byte[] pvtKey,
+      byte[] publicKey, int maxKeyLen) {
+    Preconditions.checkNotNull(pvtKey);
+    Preconditions.checkNotNull(publicKey);
+
+    this.securityConfig = new SecurityConfig(new OzoneConfiguration());
+    this.keyId = keyId;
+    this.expiryDate = expiryDate;
+    this.maxKeyLen = maxKeyLen;
+    if (pvtKey.length > maxKeyLen) {
+      throw new RuntimeException("can't create " + pvtKey.length +
+          " byte long DelegationKey. Max allowed length is " + maxKeyLen);
+    }
+    this.privateKey = SecurityUtil.getPrivateKey(pvtKey, securityConfig);
+    this.publicKey = SecurityUtil.getPublicKey(publicKey, securityConfig);
+  }
+
+  public int getKeyId() {
+    return keyId;
+  }
+
+  public long getExpiryDate() {
+    return expiryDate;
+  }
+
+  public PrivateKey getPrivateKey() {
+    return privateKey;
+  }
+
+  public PublicKey getPublicKey() {
+    return publicKey;
+  }
+
+  public int getMaxKeyLen() {
+    return maxKeyLen;
+  }
+
+  public byte[] getEncodedPrivateKey() {
+    return privateKey.getEncoded();
+  }
+
+  public byte[] getEncodedPubliceKey() {
+    return publicKey.getEncoded();
+  }
+
+  public void setExpiryDate(long expiryDate) {
+    this.expiryDate = expiryDate;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    SecretKeyProto token = SecretKeyProto.newBuilder()
+        .setKeyId(getKeyId())
+        .setExpiryDate(getExpiryDate())
+        .setPrivateKeyBytes(ByteString.copyFrom(getEncodedPrivateKey()))
+        .setPublicKeyBytes(ByteString.copyFrom(getEncodedPubliceKey()))
+        .setMaxKeyLen(getMaxKeyLen())
+        .build();
+    out.write(token.toByteArray());
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    SecretKeyProto secretKey = SecretKeyProto.parseFrom((DataInputStream) in);
+    expiryDate = secretKey.getExpiryDate();
+    keyId = secretKey.getKeyId();
+    privateKey = SecurityUtil.getPrivateKey(secretKey.getPrivateKeyBytes()
+        .toByteArray(), securityConfig);
+    publicKey = SecurityUtil.getPublicKey(secretKey.getPublicKeyBytes()
+        .toByteArray(), securityConfig);
+    maxKeyLen = secretKey.getMaxKeyLen();
+  }
+
+  @Override
+  public int hashCode() {
+    HashCodeBuilder hashCodeBuilder = new HashCodeBuilder(537, 963);
+    hashCodeBuilder.append(getExpiryDate())
+        .append(getKeyId())
+        .append(getEncodedPrivateKey())
+        .append(getEncodedPubliceKey());
+
+    return hashCodeBuilder.build();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+
+    if (obj instanceof OzoneSecretKey) {
+      OzoneSecretKey that = (OzoneSecretKey) obj;
+      return new EqualsBuilder()
+          .append(this.keyId, that.keyId)
+          .append(this.expiryDate, that.expiryDate)
+          .append(this.privateKey, that.privateKey)
+          .append(this.publicKey, that.publicKey)
+          .build();
+    }
+    return false;
+  }
+
+  /**
+   * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+   */
+  static OzoneSecretKey readProtoBuf(DataInput in) throws IOException {
+    Preconditions.checkNotNull(in);
+    SecretKeyProto key = SecretKeyProto.parseFrom((DataInputStream) in);
+    return new OzoneSecretKey(key.getKeyId(), key.getExpiryDate(),
+        key.getPrivateKeyBytes().toByteArray(),
+        key.getPublicKeyBytes().toByteArray(), key.getMaxKeyLen());
+  }
+
+  /**
+   * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+   */
+  static OzoneSecretKey readProtoBuf(byte[] identifier) throws IOException {
+    Preconditions.checkNotNull(identifier);
+    DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+        identifier));
+    return readProtoBuf(in);
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
new file mode 100644
index 0000000..d053465
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * The token identifier for Ozone Master.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneTokenIdentifier extends
+    AbstractDelegationTokenIdentifier {
+
+  public final static Text KIND_NAME = new Text("OzoneToken");
+
+  /**
+   * Create an empty delegation token identifier.
+   */
+  public OzoneTokenIdentifier() {
+    super();
+  }
+
+  /**
+   * Create a new ozone master delegation token identifier.
+   *
+   * @param owner the effective username of the token owner
+   * @param renewer the username of the renewer
+   * @param realUser the real username of the token owner
+   */
+  public OzoneTokenIdentifier(Text owner, Text renewer, Text realUser) {
+    super(owner, renewer, realUser);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public Text getKind() {
+    return KIND_NAME;
+  }
+
+  /**
+   * Default TrivialRenewer.
+   */
+  @InterfaceAudience.Private
+  public static class Renewer extends Token.TrivialRenewer {
+
+    @Override
+    protected Text getKind() {
+      return KIND_NAME;
+    }
+  }
+
+  /**
+   * Overrides default implementation to write using Protobuf.
+   *
+   * @param out output stream
+   * @throws IOException
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    OMTokenProto token = OMTokenProto.newBuilder()
+        .setOwner(getOwner().toString())
+        .setRealUser(getRealUser().toString())
+        .setRenewer(getRenewer().toString())
+        .setIssueDate(getIssueDate())
+        .setMaxDate(getMaxDate())
+        .setSequenceNumber(getSequenceNumber())
+        .setMasterKeyId(getMasterKeyId()).build();
+    out.write(token.toByteArray());
+  }
+
+  /**
+   * Overrides default implementation to read using Protobuf.
+   *
+   * @param in input stream
+   * @throws IOException
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+    setOwner(new Text(token.getOwner()));
+    setRealUser(new Text(token.getRealUser()));
+    setRenewer(new Text(token.getRenewer()));
+    setIssueDate(token.getIssueDate());
+    setMaxDate(token.getMaxDate());
+    setSequenceNumber(token.getSequenceNumber());
+    setMasterKeyId(token.getMasterKeyId());
+  }
+
+  /**
+   * Reads protobuf encoded input stream to construct {@link
+   * OzoneTokenIdentifier}.
+   */
+  public static OzoneTokenIdentifier readProtoBuf(DataInput in)
+      throws IOException {
+    OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+    OzoneTokenIdentifier identifier = new OzoneTokenIdentifier();
+    identifier.setRenewer(new Text(token.getRenewer()));
+    identifier.setOwner(new Text(token.getOwner()));
+    identifier.setRealUser(new Text(token.getRealUser()));
+    identifier.setMaxDate(token.getMaxDate());
+    identifier.setIssueDate(token.getIssueDate());
+    identifier.setSequenceNumber(token.getSequenceNumber());
+    identifier.setMasterKeyId(token.getMasterKeyId());
+    return identifier;
+  }
+
+  /**
+   * Reads protobuf encoded input stream to construct {@link
+   * OzoneTokenIdentifier}.
+   */
+  public static OzoneTokenIdentifier readProtoBuf(byte[] identifier)
+      throws IOException {
+    DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+        identifier));
+    return readProtoBuf(in);
+  }
+
+  /**
+   * Creates new instance.
+   */
+  public static OzoneTokenIdentifier newInstance() {
+    return new OzoneTokenIdentifier();
+  }
+
+  /**
+   * Creates new instance.
+   */
+  public static OzoneTokenIdentifier newInstance(Text owner, Text renewer,
+      Text realUser) {
+    return new OzoneTokenIdentifier(owner, renewer, realUser);
+  }
+
+  @Override
+  public int hashCode() {
+    return super.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof OzoneTokenIdentifier)) {
+      return false;
+    }
+    return super.equals(obj);
+  }
+
+  /**
+   * Class to encapsulate a token's renew date and password.
+   */
+  @InterfaceStability.Evolving
+  public static class TokenInfo {
+
+    private long renewDate;
+    private byte[] password;
+    private String trackingId;
+
+    public TokenInfo(long renewDate, byte[] password) {
+      this(renewDate, password, null);
+    }
+
+    public TokenInfo(long renewDate, byte[] password,
+        String trackingId) {
+      this.renewDate = renewDate;
+      this.password = Arrays.copyOf(password, password.length);
+      this.trackingId = trackingId;
+    }
+
+    /**
+     * returns renew date.
+     */
+    public long getRenewDate() {
+      return renewDate;
+    }
+
+    /**
+     * returns password.
+     */
+    byte[] getPassword() {
+      return password;
+    }
+
+    /**
+     * returns tracking id.
+     */
+    public String getTrackingId() {
+      return trackingId;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..457f891
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security related classes.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 6320cf1..5caebf5 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -311,6 +311,26 @@ message DeleteBucketResponse {
     required Status status = 1;
 }
 
+message OMTokenProto {
+    optional uint32 version        = 1;
+    optional string owner          = 2;
+    optional string renewer        = 3;
+    optional string realUser       = 4;
+    optional uint64 issueDate      = 5;
+    optional uint64 maxDate        = 6;
+    optional uint32 sequenceNumber = 7;
+    optional uint32 masterKeyId    = 8;
+    optional uint64 expiryDate     = 9;
+}
+
+message SecretKeyProto {
+    required uint32 keyId = 1;
+    required uint64 expiryDate = 2;
+    required bytes privateKeyBytes = 3;
+    required bytes publicKeyBytes = 4;
+    required uint32 maxKeyLen = 5;
+}
+
 message ListKeysRequest {
     required string volumeName = 1;
     required string bucketName = 2;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..5909916
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestOzoneBlockTokenIdentifier.class);
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+  private static final String KEYSTORES_DIR =
+      new File(BASEDIR).getAbsolutePath();
+  private static long expiryTime;
+  private static KeyPair keyPair;
+  private static X509Certificate cert;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    File base = new File(BASEDIR);
+    FileUtil.fullyDelete(base);
+    base.mkdirs();
+    expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+    // Create Ozone Master key pair.
+    keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    // Create Ozone Master certificate (SCM CA issued cert) and key store.
+    cert = KeyStoreTestUtil
+        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+  }
+
+  @After
+  public void cleanUp() throws Exception {
+    // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+  }
+
+  @Test
+  public void testSignToken() throws GeneralSecurityException, IOException {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+        .getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+        .getAbsolutePath();
+    String trustPassword = "trustPass";
+    String keyStorePassword = "keyStorePass";
+    String keyPassword = "keyPass";
+
+
+    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+        "OzoneMaster", keyPair.getPrivate(), cert);
+
+    // Create trust store and put the certificate in the trust store
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+        cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Sign the OzoneMaster Token with Ozone Master private key
+    PrivateKey privateKey = keyPair.getPrivate();
+    OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+        "testUser", "84940",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString());
+    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+    // Verify a valid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+    // Verify an invalid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    tokenId = new OzoneBlockTokenIdentifier("", "",
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString());
+    LOG.info("Unsigned token {} is {}", tokenId,
+        verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+  }
+
+  public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      PrivateKey privateKey) throws NoSuchAlgorithmException,
+      InvalidKeyException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initSign(privateKey);
+    rsaSignature.update(tokenId.getBytes());
+    byte[] signature = rsaSignature.sign();
+    return signature;
+  }
+
+  public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+      byte[] signature, Certificate certificate) throws InvalidKeyException,
+      NoSuchAlgorithmException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initVerify(certificate);
+    rsaSignature.update(tokenId.getBytes());
+    boolean isValid = rsaSignature.verify(signature);
+    return isValid;
+  }
+
+  private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+      Mac mac, SecretKey key) {
+    try {
+      mac.init(key);
+    } catch (InvalidKeyException ike) {
+      throw new IllegalArgumentException("Invalid key to HMAC computation",
+          ike);
+    }
+    return mac.doFinal(identifier.getBytes());
+  }
+
+  OzoneBlockTokenIdentifier generateTestToken() {
+    return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+        RandomStringUtils.randomAlphabetic(5),
+        EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+        expiryTime, cert.getSerialNumber().toString());
+  }
+
+  @Test
+  public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+      CertificateEncodingException, NoSuchProviderException,
+      InvalidKeyException, SignatureException {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordAsym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+    // Create Ozone Master certificate (SCM CA issued cert) and key store
+    X509Certificate certificate;
+    certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+        kp, 30, "SHA256withRSA");
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordAsym.add(
+          signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+        duration / testTokenCount);
+
+    startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+          certificate);
+    }
+    duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+        + "is {} ns", duration / testTokenCount);
+  }
+
+  @Test
+  public void testSymmetricTokenPerf() {
+    String hmacSHA1 = "HmacSHA1";
+    String hmacSHA256 = "HmacSHA256";
+
+    testSymmetricTokenPerfHelper(hmacSHA1, 64);
+    testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+  }
+
+  public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+    final int testTokenCount = 1000;
+    List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordSym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyGenerator keyGen;
+    try {
+      keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+      keyGen.init(keyLen);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    Mac mac;
+    try {
+      mac = Mac.getInstance(hmacAlgorithm);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    SecretKey secretKey = keyGen.generateKey();
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordSym.add(
+          signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+        hmacAlgorithm, keyLen, duration / testTokenCount);
+  }
+
+  // TODO: verify certificate with a trust store
+  public boolean verifyCert(Certificate certificate) {
+    return true;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
new file mode 100644
index 0000000..dfdf14b
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.TestSSLFactory;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for {@link OzoneTokenIdentifier}.
+ */
+public class TestOzoneTokenIdentifier {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestOzoneTokenIdentifier.class);
+  private static final String BASEDIR = GenericTestUtils
+      .getTempPath(TestOzoneTokenIdentifier.class.getSimpleName());
+  private static final String KEYSTORES_DIR =
+      new File(BASEDIR).getAbsolutePath();
+  private static File base;
+  private static String sslConfsDir;
+  private static final String EXCLUDE_CIPHERS =
+      "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+      + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,  \n"
+      + "SSL_RSA_WITH_DES_CBC_SHA,"
+      + "SSL_DHE_RSA_WITH_DES_CBC_SHA,  "
+      + "SSL_RSA_EXPORT_WITH_RC4_40_MD5,\t \n"
+      + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+      + "SSL_RSA_WITH_RC4_128_MD5";
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    base = new File(BASEDIR);
+    FileUtil.fullyDelete(base);
+    base.mkdirs();
+  }
+
+  private Configuration createConfiguration(boolean clientCert,
+                                            boolean trustStore)
+      throws Exception {
+    Configuration conf = new Configuration();
+    KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
+        clientCert, trustStore, EXCLUDE_CIPHERS);
+    sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
+    return conf;
+  }
+
+  @AfterClass
+  static public void cleanUp() throws Exception {
+    FileUtil.fullyDelete(base);
+    KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+  }
+
+  @Test
+  public void testSignToken() throws GeneralSecurityException, IOException {
+    String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+        .getAbsolutePath();
+    String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+        .getAbsolutePath();
+    String trustPassword = "trustPass";
+    String keyStorePassword = "keyStorePass";
+    String keyPassword = "keyPass";
+
+    // Create Ozone Master key pair
+    KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+    // Create Ozone Master certificate (SCM CA issued cert) and key store
+    X509Certificate cert = KeyStoreTestUtil
+        .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+    KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+        "OzoneMaster", keyPair.getPrivate(), cert);
+
+    // Create trust store and put the certificate in the trust store
+    Map<String, X509Certificate> certs = Collections.singletonMap("server",
+        cert);
+    KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+    // Sign the OzoneMaster Token with Ozone Master private key
+    PrivateKey privateKey = keyPair.getPrivate();
+    OzoneTokenIdentifier tokenId = new OzoneTokenIdentifier();
+    byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+    // Verify a valid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+    LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+    // Verify an invalid signed OzoneMaster Token with Ozone Master
+    // public key(certificate)
+    tokenId = new OzoneTokenIdentifier(new Text("oozie"),
+        new Text("rm"), new Text("client"));
+    LOG.info("Unsigned token {} is {}", tokenId,
+        verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+  }
+
+  public byte[] signTokenAsymmetric(OzoneTokenIdentifier tokenId,
+      PrivateKey privateKey) throws NoSuchAlgorithmException,
+      InvalidKeyException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initSign(privateKey);
+    rsaSignature.update(tokenId.getBytes());
+    byte[] signature = rsaSignature.sign();
+    return signature;
+  }
+
+  public boolean verifyTokenAsymmetric(OzoneTokenIdentifier tokenId,
+      byte[] signature, Certificate certificate) throws InvalidKeyException,
+      NoSuchAlgorithmException, SignatureException {
+    Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+    rsaSignature.initVerify(certificate);
+    rsaSignature.update(tokenId.getBytes());
+    boolean isValide = rsaSignature.verify(signature);
+    return isValide;
+  }
+
+  private byte[] signTokenSymmetric(OzoneTokenIdentifier identifier,
+      Mac mac, SecretKey key) {
+    try {
+      mac.init(key);
+    } catch (InvalidKeyException ike) {
+      throw new IllegalArgumentException("Invalid key to HMAC computation",
+          ike);
+    }
+    return mac.doFinal(identifier.getBytes());
+  }
+
+  OzoneTokenIdentifier generateTestToken() {
+    return new OzoneTokenIdentifier(
+        new Text(RandomStringUtils.randomAlphabetic(6)),
+        new Text(RandomStringUtils.randomAlphabetic(5)),
+        new Text(RandomStringUtils.randomAlphabetic(4)));
+  }
+
+  @Test
+  public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+      CertificateEncodingException, NoSuchProviderException,
+      InvalidKeyException, SignatureException {
+    final int testTokenCount = 1000;
+    List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordAsym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+    // Create Ozone Master certificate (SCM CA issued cert) and key store
+    X509Certificate cert;
+    cert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+        keyPair, 30, "SHA256withRSA");
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordAsym.add(
+          signTokenAsymmetric(tokenIds.get(i), keyPair.getPrivate()));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+        duration/testTokenCount);
+
+    startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), cert);
+    }
+    duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+            + "is {} ns", duration/testTokenCount);
+  }
+
+  @Test
+  public void testSymmetricTokenPerf() {
+    String hmacSHA1 = "HmacSHA1";
+    String hmacSHA256 = "HmacSHA256";
+
+    testSymmetricTokenPerfHelper(hmacSHA1, 64);
+    testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+  }
+
+
+  public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+    final int testTokenCount = 1000;
+    List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+    List<byte[]> tokenPasswordSym = new ArrayList<>();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenIds.add(generateTestToken());
+    }
+
+    KeyGenerator keyGen;
+    try {
+      keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+      keyGen.init(keyLen);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    Mac mac;
+    try {
+      mac =  Mac.getInstance(hmacAlgorithm);
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+          " algorithm.");
+    }
+
+    SecretKey secretKey = keyGen.generateKey();
+
+    long startTime = Time.monotonicNowNanos();
+    for (int i = 0; i < testTokenCount; i++) {
+      tokenPasswordSym.add(
+          signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+    }
+    long duration = Time.monotonicNowNanos() - startTime;
+    LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+        hmacAlgorithm, keyLen, duration/testTokenCount);
+  }
+
+  /*
+   * Test serialization/deserialization of OzoneTokenIdentifier.
+   */
+  @Test
+  public void testReadWriteInProtobuf() throws IOException {
+    OzoneTokenIdentifier id = getIdentifierInst();
+    File idFile = new File(BASEDIR + "/tokenFile");
+
+    FileOutputStream fop = new FileOutputStream(idFile);
+    DataOutputStream dataOutputStream = new DataOutputStream(fop);
+    id.write(dataOutputStream);
+    fop.close();
+
+    FileInputStream fis = new FileInputStream(idFile);
+    DataInputStream dis = new DataInputStream(fis);
+    OzoneTokenIdentifier id2 = new OzoneTokenIdentifier();
+
+    id2.readFields(dis);
+    Assert.assertEquals(id, id2);
+  }
+
+
+  public OzoneTokenIdentifier getIdentifierInst() {
+    OzoneTokenIdentifier id = new OzoneTokenIdentifier();
+    id.setOwner(new Text("User1"));
+    id.setRenewer(new Text("yarn"));
+    id.setIssueDate(Time.now());
+    id.setMaxDate(Time.now() + 5000);
+    id.setSequenceNumber(1);
+    return id;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..a36f325
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security tests.
+ */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[16/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcd705a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcd705a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcd705a5

Branch: refs/heads/HDDS-4
Commit: fcd705a59420dcb5d6a05e2d49536a77ae4f1d5b
Parents: 2119be4
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800

----------------------------------------------------------------------
 hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++++
 1 file changed, 4 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd705a5/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index d3e352b..4d79e8c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -198,6 +198,7 @@
     </description>
   </property>
   <property>
+<<<<<<< HEAD
     <name>dfs.ratis.client.request.timeout.duration</name>
     <value>3s</value>
     <tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -255,6 +256,9 @@
   </property>
   <property>
     <name>hdds.container.report.interval</name>
+=======
+    <name>ozone.container.report.interval</name>
+>>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
     <value>60000ms</value>
     <tag>OZONE, CONTAINER, MANAGEMENT</tag>
     <description>Time interval of the datanode to send container report. Each


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[23/50] [abbrv] hadoop git commit: HDDS-101. SCM CA: generate CSR for SCM CA clients. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-101. SCM CA: generate CSR for SCM CA clients.
Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1842ced1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1842ced1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1842ced1

Branch: refs/heads/HDDS-4
Commit: 1842ced1268acdf59accb54aec664043bf07da73
Parents: 7246898
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Oct 26 17:57:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../certificates/CertificateSignRequest.java    | 245 +++++++++++++++++
 .../security/x509/keys/HDDSKeyGenerator.java    |   2 +-
 .../hdds/security/x509/keys/SecurityUtil.java   |  79 ++++++
 .../TestCertificateSignRequest.java             | 268 +++++++++++++++++++
 4 files changed, 593 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
new file mode 100644
index 0000000..2e1f9df
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CA.
+ */
+public final class CertificateSignRequest {
+  private final KeyPair keyPair;
+  private final SecurityConfig config;
+  private final Extensions extensions;
+  private String subject;
+  private String clusterID;
+  private String scmID;
+
+  /**
+   * Private Ctor for CSR.
+   *
+   * @param subject - Subject
+   * @param scmID - SCM ID
+   * @param clusterID - Cluster ID
+   * @param keyPair - KeyPair
+   * @param config - SCM Config
+   * @param extensions - CSR extensions
+   */
+  private CertificateSignRequest(String subject, String scmID, String clusterID,
+      KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+    this.keyPair = keyPair;
+    this.config = config;
+    this.extensions = extensions;
+  }
+
+  private PKCS10CertificationRequest generateCSR() throws
+      OperatorCreationException {
+    X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+        clusterID);
+    PKCS10CertificationRequestBuilder p10Builder =
+        new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+    ContentSigner contentSigner =
+        new JcaContentSignerBuilder(config.getSignatureAlgo())
+            .setProvider(config.getProvider())
+            .build(keyPair.getPrivate());
+
+    if (extensions != null) {
+      p10Builder.addAttribute(
+          PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+    }
+    return p10Builder.build(contentSigner);
+  }
+
+  /**
+   * Builder class for Certificate Sign Request.
+   */
+  public static class Builder {
+    private String subject;
+    private String clusterID;
+    private String scmID;
+    private KeyPair key;
+    private SecurityConfig config;
+    private List<GeneralName> altNames;
+    private Boolean ca = false;
+
+    public CertificateSignRequest.Builder setConfiguration(
+        Configuration configuration) {
+      this.config = new SecurityConfig(configuration);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+      this.key = keyPair;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setSubject(String subjectString) {
+      this.subject = subjectString;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setClusterID(String s) {
+      this.clusterID = s;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setScmID(String s) {
+      this.scmID = s;
+      return this;
+    }
+
+    // Support SAN extenion with DNS and RFC822 Name
+    // other name type will be added as needed.
+    public CertificateSignRequest.Builder addDnsName(String dnsName) {
+      Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+      this.addAltName(GeneralName.dNSName, dnsName);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder addRfc822Name(String name) {
+      Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+      this.addAltName(GeneralName.rfc822Name, name);
+      return this;
+    }
+
+    // IP address is subject to change which is optional for now.
+    public CertificateSignRequest.Builder addIpAddress(String ip) {
+      Preconditions.checkNotNull(ip, "Ip address cannot be null");
+      this.addAltName(GeneralName.iPAddress, ip);
+      return this;
+    }
+
+    private CertificateSignRequest.Builder addAltName(int tag, String name) {
+      if (altNames == null) {
+        altNames = new ArrayList<>();
+      }
+      altNames.add(new GeneralName(tag, name));
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setCA(Boolean isCA) {
+      this.ca = isCA;
+      return this;
+    }
+
+    private Extension getKeyUsageExtension() throws IOException {
+      int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+          | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+      if (ca) {
+        keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+      }
+      KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+      return new Extension(Extension.keyUsage, true,
+          new DEROctetString(keyUsage));
+    }
+
+    private Optional<Extension> getSubjectAltNameExtension() throws
+        IOException {
+      if (altNames != null) {
+        return Optional.of(new Extension(Extension.subjectAlternativeName,
+            true, new DEROctetString(new GeneralNames(
+            altNames.toArray(new GeneralName[altNames.size()])))));
+      }
+      return Optional.empty();
+    }
+
+    private Extension getBasicExtension() throws IOException {
+      // We don't set pathLenConstraint means no limit is imposed.
+      return new Extension(Extension.basicConstraints,
+          true, new DEROctetString(new BasicConstraints(ca)));
+    }
+
+    private Extensions createExtensions() throws IOException {
+      List<Extension> extensions = new ArrayList<>();
+
+      // Add basic extension
+      extensions.add(getBasicExtension());
+
+      // Add key usage extension
+      extensions.add(getKeyUsageExtension());
+
+      // Add subject alternate name extension
+      Optional<Extension> san = getSubjectAltNameExtension();
+      if (san.isPresent()) {
+        extensions.add(san.get());
+      }
+
+      return new Extensions(
+          extensions.toArray(new Extension[extensions.size()]));
+    }
+
+    public PKCS10CertificationRequest build() throws SCMSecurityException {
+      Preconditions.checkNotNull(key, "KeyPair cannot be null");
+      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+          "be blank");
+
+      try {
+        CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+            clusterID, key, config, createExtensions());
+        return csr.generateCSR();
+      } catch (IOException ioe) {
+        throw new CertificateException(String.format("Unable to create " +
+            "extension for certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+      } catch (OperatorCreationException ex) {
+        throw new CertificateException(String.format("Unable to create " +
+            "certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)),
+            ex.getCause());
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 99873cb..459dce7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -96,7 +96,7 @@ public class HDDSKeyGenerator {
    */
   public KeyPair generateKey(int size, String algorithm, String provider)
       throws NoSuchProviderException, NoSuchAlgorithmException {
-    LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+    LOG.debug("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
         size, algorithm, provider);
     KeyPairGenerator generator = KeyPairGenerator
         .getInstance(algorithm, provider);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
new file mode 100644
index 0000000..2ca8825
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.ASN1Set;
+import org.bouncycastle.asn1.pkcs.Attribute;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+
+/**
+ * Utility functions for Security modules for Ozone.
+ */
+public final class SecurityUtil {
+
+  // Ozone Certificate distinguished format: (CN=Subject,OU=ScmID,O=ClusterID).
+  private static final String DISTINGUISHED_NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+
+  private SecurityUtil() {
+  }
+
+  public static String getDistinguishedNameFormat() {
+    return DISTINGUISHED_NAME_FORMAT;
+  }
+
+  public static X500Name getDistinguishedName(String subject, String scmID,
+      String clusterID) {
+    return new X500Name(String.format(getDistinguishedNameFormat(), subject,
+        scmID, clusterID));
+  }
+
+  // TODO: move the PKCS10CSRValidator class
+  public static Extensions getPkcs9Extensions(PKCS10CertificationRequest csr)
+      throws CertificateException {
+    ASN1Set pkcs9ExtReq = getPkcs9ExtRequest(csr);
+    Object extReqElement = pkcs9ExtReq.getObjects().nextElement();
+    if (extReqElement instanceof Extensions) {
+      return (Extensions) extReqElement;
+    } else {
+      if (extReqElement instanceof ASN1Sequence) {
+        return Extensions.getInstance((ASN1Sequence) extReqElement);
+      } else {
+        throw new CertificateException("Unknown element type :" + extReqElement
+            .getClass().getSimpleName());
+      }
+    }
+  }
+
+  public static ASN1Set getPkcs9ExtRequest(PKCS10CertificationRequest csr)
+      throws CertificateException {
+    for (Attribute attr : csr.getAttributes()) {
+      ASN1ObjectIdentifier oid = attr.getAttrType();
+      if (oid.equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) {
+        return attr.getAttrValues();
+      }
+    }
+    throw new CertificateException("No PKCS#9 extension found in CSR");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
new file mode 100644
index 0000000..a9285df
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -0,0 +1,268 @@
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.operator.ContentVerifierProvider;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCSException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+public class TestCertificateSignRequest {
+
+  private SecurityConfig securityConfig;
+  private static OzoneConfiguration conf = new OzoneConfiguration();
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Before
+  public void init() throws IOException {
+    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+    securityConfig = new SecurityConfig(conf);
+  }
+
+  @Test
+  public void testGenerateCSR() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException,
+      OperatorCreationException, PKCSException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+    Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+    // Verify basic constraints extension
+    Extension basicExt = extensions.getExtension(Extension
+        .basicConstraints);
+    Assert.assertEquals(true, basicExt.isCritical());
+
+    // Verify key usage extension
+    Extension keyUsageExt = extensions.getExtension(Extension.keyUsage);
+    Assert.assertEquals(true, keyUsageExt.isCritical());
+
+
+    // Verify San extension not set
+    Assert.assertEquals(null,
+        extensions.getExtension(Extension.subjectAlternativeName));
+
+    // Verify signature in CSR
+    ContentVerifierProvider verifierProvider =
+        new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+            .getProvider()).build(csr.getSubjectPublicKeyInfo());
+    Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+  }
+
+  @Test
+  public void testGenerateCSRwithSan() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException,
+      OperatorCreationException, PKCSException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+
+    // Multi-home
+    builder.addIpAddress("192.168.1.1");
+    builder.addIpAddress("192.168.2.1");
+
+    builder.addDnsName("dn1.abc.com");
+    builder.addRfc822Name("test@abc.com");
+
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+    Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+    // Verify key usage extension
+    Extension sanExt = extensions.getExtension(Extension.keyUsage);
+    Assert.assertEquals(true, sanExt.isCritical());
+
+
+    // Verify signature in CSR
+    ContentVerifierProvider verifierProvider =
+        new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+            .getProvider()).build(csr.getSubjectPublicKeyInfo());
+    Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+  }
+
+  @Test
+  public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+
+    try {
+      builder.setKey(null);
+      builder.build();
+      Assert.fail("Null Key should have failed.");
+    } catch (NullPointerException | IllegalArgumentException e) {
+      builder.setKey(keyPair);
+    }
+
+    // Now try with blank/null Subject.
+    try {
+      builder.setSubject(null);
+      builder.build();
+      Assert.fail("Null/Blank Subject should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setSubject(subject);
+    }
+
+    try {
+      builder.setSubject("");
+      builder.build();
+      Assert.fail("Null/Blank Subject should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setSubject(subject);
+    }
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setScmID(null);
+      builder.build();
+      Assert.fail("Null/Blank SCM ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setScmID(scmID);
+    }
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setClusterID(null);
+      builder.build();
+      Assert.fail("Null/Blank Cluster ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setClusterID(clusterID);
+    }
+
+    // Now try with invalid IP address
+    try {
+      builder.addIpAddress("255.255.255.*");
+      builder.build();
+      Assert.fail("Invalid ip address");
+    } catch (IllegalArgumentException e) {
+    }
+
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+  }
+  
+  @Test
+  public void testCsrSerialization() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException, IOException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+    PKCS10CertificationRequest csr = builder.build();
+    byte[] csrBytes = csr.getEncoded();
+
+    // Verify de-serialized CSR matches with the original CSR
+    PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes);
+    Assert.assertEquals(csr, dsCsr);
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[10/50] [abbrv] hadoop git commit: HDDS-808. Simplify OMAction and DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.

Posted by xy...@apache.org.
HDDS-808. Simplify OMAction and DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/184cced5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/184cced5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/184cced5

Branch: refs/heads/HDDS-4
Commit: 184cced513c5599d7b33c9124692fbcd2e6d338e
Parents: 07142f5
Author: Ajay Kumar <aj...@apache.org>
Authored: Thu Nov 29 08:35:02 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Thu Nov 29 08:35:20 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/ozone/audit/DNAction.java | 44 +++++++---------
 .../apache/hadoop/ozone/audit/DummyAction.java  | 36 ++++++-------
 .../org/apache/hadoop/ozone/audit/OMAction.java | 54 +++++++++-----------
 3 files changed, 58 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
index ce34c46..1c87f2b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -21,34 +21,28 @@ package org.apache.hadoop.ozone.audit;
  */
 public enum DNAction implements AuditAction {
 
-  CREATE_CONTAINER("CREATE_CONTAINER"),
-  READ_CONTAINER("READ_CONTAINER"),
-  UPDATE_CONTAINER("UPDATE_CONTAINER"),
-  DELETE_CONTAINER("DELETE_CONTAINER"),
-  LIST_CONTAINER("LIST_CONTAINER"),
-  PUT_BLOCK("PUT_BLOCK"),
-  GET_BLOCK("GET_BLOCK"),
-  DELETE_BLOCK("DELETE_BLOCK"),
-  LIST_BLOCK("LIST_BLOCK"),
-  READ_CHUNK("READ_CHUNK"),
-  DELETE_CHUNK("DELETE_CHUNK"),
-  WRITE_CHUNK("WRITE_CHUNK"),
-  LIST_CHUNK("LIST_CHUNK"),
-  COMPACT_CHUNK("COMPACT_CHUNK"),
-  PUT_SMALL_FILE("PUT_SMALL_FILE"),
-  GET_SMALL_FILE("GET_SMALL_FILE"),
-  CLOSE_CONTAINER("CLOSE_CONTAINER"),
-  GET_COMMITTED_BLOCK_LENGTH("GET_COMMITTED_BLOCK_LENGTH");
-
-  private String action;
-
-  DNAction(String action) {
-    this.action = action;
-  }
+  CREATE_CONTAINER,
+  READ_CONTAINER,
+  UPDATE_CONTAINER,
+  DELETE_CONTAINER,
+  LIST_CONTAINER,
+  PUT_BLOCK,
+  GET_BLOCK,
+  DELETE_BLOCK,
+  LIST_BLOCK,
+  READ_CHUNK,
+  DELETE_CHUNK,
+  WRITE_CHUNK,
+  LIST_CHUNK,
+  COMPACT_CHUNK,
+  PUT_SMALL_FILE,
+  GET_SMALL_FILE,
+  CLOSE_CONTAINER,
+  GET_COMMITTED_BLOCK_LENGTH;
 
   @Override
   public String getAction() {
-    return this.action;
+    return this.toString();
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
index 76cd39a..d2da3e6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
@@ -22,30 +22,24 @@ package org.apache.hadoop.ozone.audit;
  */
 public enum DummyAction implements AuditAction {
 
-  CREATE_VOLUME("CREATE_VOLUME"),
-  CREATE_BUCKET("CREATE_BUCKET"),
-  CREATE_KEY("CREATE_KEY"),
-  READ_VOLUME("READ_VOLUME"),
-  READ_BUCKET("READ_BUCKET"),
-  READ_KEY("READ_BUCKET"),
-  UPDATE_VOLUME("UPDATE_VOLUME"),
-  UPDATE_BUCKET("UPDATE_BUCKET"),
-  UPDATE_KEY("UPDATE_KEY"),
-  DELETE_VOLUME("DELETE_VOLUME"),
-  DELETE_BUCKET("DELETE_BUCKET"),
-  DELETE_KEY("DELETE_KEY"),
-  SET_OWNER("SET_OWNER"),
-  SET_QUOTA("SET_QUOTA");
-
-  private final String action;
-
-  DummyAction(String action) {
-    this.action = action;
-  }
+  CREATE_VOLUME,
+  CREATE_BUCKET,
+  CREATE_KEY,
+  READ_VOLUME,
+  READ_BUCKET,
+  READ_KEY,
+  UPDATE_VOLUME,
+  UPDATE_BUCKET,
+  UPDATE_KEY,
+  DELETE_VOLUME,
+  DELETE_BUCKET,
+  DELETE_KEY,
+  SET_OWNER,
+  SET_QUOTA;
 
   @Override
   public String getAction() {
-    return this.action;
+    return this.toString();
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index 1d4d646..8794014 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -23,40 +23,34 @@ package org.apache.hadoop.ozone.audit;
 public enum OMAction implements AuditAction {
 
   // WRITE Actions
-  ALLOCATE_BLOCK("ALLOCATE_BLOCK"),
-  ALLOCATE_KEY("ALLOCATE_KEY"),
-  COMMIT_KEY("COMMIT_KEY"),
-  CREATE_VOLUME("CREATE_VOLUME"),
-  CREATE_BUCKET("CREATE_BUCKET"),
-  CREATE_KEY("CREATE_KEY"),
-  DELETE_VOLUME("DELETE_VOLUME"),
-  DELETE_BUCKET("DELETE_BUCKET"),
-  DELETE_KEY("DELETE_KEY"),
-  RENAME_KEY("RENAME_KEY"),
-  SET_OWNER("SET_OWNER"),
-  SET_QUOTA("SET_QUOTA"),
-  UPDATE_VOLUME("UPDATE_VOLUME"),
-  UPDATE_BUCKET("UPDATE_BUCKET"),
-  UPDATE_KEY("UPDATE_KEY"),
+  ALLOCATE_BLOCK,
+  ALLOCATE_KEY,
+  COMMIT_KEY,
+  CREATE_VOLUME,
+  CREATE_BUCKET,
+  CREATE_KEY,
+  DELETE_VOLUME,
+  DELETE_BUCKET,
+  DELETE_KEY,
+  RENAME_KEY,
+  SET_OWNER,
+  SET_QUOTA,
+  UPDATE_VOLUME,
+  UPDATE_BUCKET,
+  UPDATE_KEY,
   // READ Actions
-  CHECK_VOLUME_ACCESS("CHECK_VOLUME_ACCESS"),
-  LIST_BUCKETS("LIST_BUCKETS"),
-  LIST_VOLUMES("LIST_VOLUMES"),
-  LIST_KEYS("LIST_KEYS"),
-  READ_VOLUME("READ_VOLUME"),
-  READ_BUCKET("READ_BUCKET"),
-  READ_KEY("READ_BUCKET"),
-  LIST_S3BUCKETS("LIST_S3BUCKETS");
-
-  private String action;
-
-  OMAction(String action) {
-    this.action = action;
-  }
+  CHECK_VOLUME_ACCESS,
+  LIST_BUCKETS,
+  LIST_VOLUMES,
+  LIST_KEYS,
+  READ_VOLUME,
+  READ_BUCKET,
+  READ_KEY,
+  LIST_S3BUCKETS;
 
   @Override
   public String getAction() {
-    return this.action;
+    return this.toString();
   }
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[25/50] [abbrv] hadoop git commit: Fix merge conflicts

Posted by xy...@apache.org.
Fix merge conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b802cb5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b802cb5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b802cb5b

Branch: refs/heads/HDDS-4
Commit: b802cb5b039172111a815f9ba8c8415f2d402fb8
Parents: ed10fa6
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Jul 31 18:17:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../conf/TestConfigurationFieldsBase.java       |  2 -
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   | 10 ++-
 .../apache/hadoop/ozone/OzoneConfigKeys.java    |  1 +
 .../common/src/main/resources/ozone-default.xml | 21 ++---
 .../scm/server/StorageContainerManager.java     |  9 ++-
 .../StorageContainerManagerHttpServer.java      |  4 +-
 .../src/test/compose/compose-secure/.env        |  2 +-
 .../test/compose/compose-secure/docker-config   | 55 ++++++++++---
 .../apache/hadoop/ozone/ksm/KSMConfigKeys.java  | 84 --------------------
 .../apache/hadoop/ozone/MiniOzoneCluster.java   |  3 +-
 .../hadoop/ozone/MiniOzoneClusterImpl.java      |  5 +-
 .../hadoop/ozone/TestSecureOzoneCluster.java    | 22 ++---
 12 files changed, 87 insertions(+), 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index bce1cd5..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,8 +436,6 @@ public abstract class TestConfigurationFieldsBase {
     // Create XML key/value map
     LOG_XML.debug("Reading XML property files\n");
     xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
-    // Remove hadoop property set in ozone-default.xml
-    xmlKeyValueMap.remove("hadoop.custom.tags");
     LOG_XML.debug("\n=====\n");
 
     // Create default configuration variable key/value map

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 376e6db..cb6672f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -313,10 +313,12 @@ public final class ScmConfigKeys {
   public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
       "10m";
 
-  public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
-      "ozone.scm.web.authentication.kerberos.principal";
-  public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
-      "ozone.scm.web.authentication.kerberos.keytab";
+  public static final String
+      HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+      "hdds.scm.web.authentication.kerberos.principal";
+  public static final String
+      HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+      "hdds.scm.web.authentication.kerberos.keytab";
   /**
    * Never constructed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 578a983..b47113b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -334,6 +334,7 @@ public final class OzoneConfigKeys {
 
   public static final String OZONE_CONTAINER_COPY_WORKDIR =
       "hdds.datanode.replication.work.dir";
+
   /**
    * Config properties to set client side checksum properties.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0c99368..7d91dc8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -500,7 +500,6 @@
     </description>
   </property>
   <property>
-<<<<<<< HEAD
     <name>ozone.om.keytab.file</name>
     <value/>
     <tag>OM, SECURITY</tag>
@@ -510,9 +509,6 @@
   </property>
   <property>
     <name>ozone.om.db.cache.size.mb</name>
-=======
-    <name>ozone.ksm.db.cache.size.mb</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
     <value>128</value>
     <tag>OM, PERFORMANCE</tag>
     <description>
@@ -1023,7 +1019,7 @@
   </property>
 
   <property>
-    <name>hdds.scm.container.creation.lease.timeout</name>
+    <name>ozone.scm.container.creation.lease.timeout</name>
     <value>60s</value>
     <tag>OZONE, SCM</tag>
     <description>
@@ -1077,11 +1073,7 @@
     </description>
   </property>
   <property>
-<<<<<<< HEAD
     <name>hdds.container.close.threshold</name>
-=======
-    <name>hdds.scm.container.close.threshold</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
     <value>0.9f</value>
     <tag>OZONE, DATANODE</tag>
     <description>
@@ -1285,8 +1277,8 @@
       datanode unless the datanode confirms the completion.
     </description>
   </property>
+
   <property>
-<<<<<<< HEAD
     <name>hdds.db.profile</name>
     <value>DISK</value>
     <tag>OZONE, OM, PERFORMANCE</tag>
@@ -1294,6 +1286,7 @@
     that tunes the RocksDB settings for the hardware it is running
     on. Right now, we have SSD and DISK as profile options.</description>
   </property>
+
   <property>
     <name>hdds.datanode.replication.work.dir</name>
     <tag>DATANODE</tag>
@@ -1395,6 +1388,14 @@
       file. Unit could be defined with postfix (ns,ms,s,m,h,d)
     </description>
   </property>
+  <property>
+    <name>ozone.security.enabled</name>
+    <value>false</value>
+    <tag> OZONE, SECURITY, FLAG</tag>
+    <description>True if security is enabled for ozone. When this property is
+     true, hadoop.security.authentication should be Kerberos.
+    </description>
+  </property>
 
   <property>
     <name>hdds.scm.kerberos.keytab.file</name>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 27fe326..2d5b330 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -442,11 +442,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
    * @param args command line arguments.
    * @param conf HDDS configuration
    * @return SCM instance
-   * @throws IOException
+   * @throws IOException, AuthenticationException
    */
   @VisibleForTesting
-  public static StorageContainerManager createSCM(String[] args,
-      OzoneConfiguration conf) throws IOException, AuthenticationException {
+  public static StorageContainerManager createSCM(
+      String[] args, OzoneConfiguration conf)
+      throws IOException, AuthenticationException {
     return createSCM(args, conf, false);
   }
 
@@ -457,7 +458,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
    * @param conf HDDS configuration
    * @param printBanner if true, then log a verbose startup message.
    * @return SCM instance
-   * @throws IOException
+   * @throws IOException, AuthenticationException
    */
   private static StorageContainerManager createSCM(
       String[] args,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index da936ad..41dd89a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
   }
 
   @Override protected String getKeytabFile() {
-    return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+    return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
   }
 
   @Override protected String getSpnegoPrincipal() {
-    return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+    return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
   }
 
   @Override protected String getEnabledKey() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
index 3254735..199d52c 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
+OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 360b69a..cdf72ce 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -22,7 +22,7 @@ OZONE-SITE.XML_hdds.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.replication=1
 OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
 OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
 OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
@@ -32,21 +32,12 @@ OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.C
 OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
 OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
 OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.block.access.token.enable=true
-HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
 HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
 HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
 HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
 HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
 HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
 CORE-SITE.XML_dfs.data.transfer.protection=authentication
 CORE-SITE.XML_hadoop.security.authentication=kerberos
 CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
@@ -55,9 +46,51 @@ LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
 
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+
 OZONE_DATANODE_SECURE_USER=root
 CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn om scm HTTP testuser
+KERBEROS_KEYTABS=dn om scm HTTP testuser
 KERBEROS_KEYSTORES=hadoop
 KERBEROS_SERVER=ozone.kdc
 JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index cc25dbe..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
-  /**
-   * Never constructed.
-   */
-  private KSMConfigKeys() {
-  }
-
-
-  public static final String OZONE_KSM_HANDLER_COUNT_KEY =
-      "ozone.ksm.handler.count.key";
-  public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
-  public static final String OZONE_KSM_ADDRESS_KEY =
-      "ozone.ksm.address";
-  public static final String OZONE_KSM_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-  public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
-  public static final String OZONE_KSM_HTTP_ENABLED_KEY =
-      "ozone.ksm.http.enabled";
-  public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
-      "ozone.ksm.http-bind-host";
-  public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
-      "ozone.ksm.https-bind-host";
-  public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
-      "ozone.ksm.http-address";
-  public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
-      "ozone.ksm.https-address";
-  public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
-  public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
-  public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
-  // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
-  public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
-      "ozone.ksm.db.cache.size.mb";
-  public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
-  public static final String OZONE_KSM_USER_MAX_VOLUME =
-      "ozone.ksm.user.max.volume";
-  public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
-  // KSM Default user/group permissions
-  public static final String OZONE_KSM_USER_RIGHTS =
-      "ozone.ksm.user.rights";
-  public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
-      OzoneAcl.OzoneACLRights.READ_WRITE;
-
-  public static final String OZONE_KSM_GROUP_RIGHTS =
-      "ozone.ksm.group.rights";
-  public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
-      OzoneAcl.OzoneACLRights.READ_WRITE;
-
-  public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
-      "ozone.key.deleting.limit.per.task";
-  public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-
-  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
-      "ozone.om.web.authentication.kerberos.principal";
-  public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
-      "ozone.om.web.authentication.kerberos.keytab";
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3fe5b76..cbed243 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -150,7 +150,8 @@ public interface MiniOzoneCluster {
    * @throws TimeoutException
    * @throws InterruptedException
    */
-  void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
+  void restartStorageContainerManager() throws InterruptedException,
+      TimeoutException, IOException, AuthenticationException;
 
   /**
    * Restarts OzoneManager instance.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 44cbecc..d7f74b9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -233,8 +233,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
   }
 
   @Override
-  public void restartStorageContainerManager() throws TimeoutException,
-      InterruptedException, IOException, AuthenticationException {
+  public void restartStorageContainerManager()
+      throws TimeoutException, InterruptedException, IOException,
+      AuthenticationException {
     scm.stop();
     scm.join();
     scm = StorageContainerManager.createSCM(null, conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 3fb5499..14c6800 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -89,10 +89,6 @@ public final class TestSecureOzoneCluster {
       startMiniKdc();
       setSecureConfig(conf);
       createCredentialsInKDC(conf, miniKdc);
-
-      clusterId = UUID.randomUUID().toString();
-      scmId = UUID.randomUUID().toString();
-      omId = UUID.randomUUID().toString();
     } catch (IOException e) {
       LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
     } catch (Exception e) {
@@ -120,11 +116,12 @@ public final class TestSecureOzoneCluster {
     createPrincipal(scmKeytab,
         conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
      createPrincipal(spnegoKeytab,
-         conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
-         conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
-    createPrincipal(omKeyTab,
+         conf.get(ScmConfigKeys
+             .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
         conf.get(OMConfigKeys
-            .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+            .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+    createPrincipal(omKeyTab,
+        conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
   }
 
   private void createPrincipal(File keytab, String... principal)
@@ -156,7 +153,7 @@ public final class TestSecureOzoneCluster {
 
     conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
         "scm/" + host + "@" + realm);
-    conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
         "HTTP_SCM/" + host + "@" + realm);
 
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
@@ -170,7 +167,7 @@ public final class TestSecureOzoneCluster {
 
     conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
         scmKeytab.getAbsolutePath());
-    conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+    conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
         spnegoKeytab.getAbsolutePath());
     conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
         omKeyTab.getAbsolutePath());
@@ -189,6 +186,11 @@ public final class TestSecureOzoneCluster {
 
   private void initSCM()
       throws IOException, AuthenticationException {
+
+    clusterId = UUID.randomUUID().toString();
+    scmId = UUID.randomUUID().toString();
+    omId = UUID.randomUUID().toString();
+
     final String path = GenericTestUtils
         .getTempPath(UUID.randomUUID().toString());
     Path scmPath = Paths.get(path, "scm-meta");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[34/50] [abbrv] hadoop git commit: HDDS-591. Adding ASF license header to kadm5.acl. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-591. Adding ASF license header to kadm5.acl. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c0280d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c0280d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c0280d

Branch: refs/heads/HDDS-4
Commit: b4c0280d1126eca2190dc40988695f7746429899
Parents: 602aa80
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 10 10:01:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../docker-image/docker-krb5/Dockerfile-krb5     |  1 +
 .../docker-image/docker-krb5/kadm5.acl           | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c0280d/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
index b5b931d..14532d4 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -28,6 +28,7 @@ RUN kdb5_util create -s -P Welcome1
 RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
 RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
 ADD launcher.sh .
+RUN chmod +x /opt/launcher.sh
 RUN mkdir -p /data
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c0280d/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
index 8fe9f69..f0cd660 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -1 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 */admin@EXAMPLE.COM x


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[41/50] [abbrv] hadoop git commit: HDDS-753. SCM security protocol server is not starting. Contributed by Ajay Kumar.

Posted by xy...@apache.org.
HDDS-753. SCM security protocol server is not starting. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9128c665
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9128c665
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9128c665

Branch: refs/heads/HDDS-4
Commit: 9128c665ccdf26eb3152d6903b98e779585acb2d
Parents: cd470bf
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 1 15:48:52 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800

----------------------------------------------------------------------
 .../scm/server/SCMSecurityProtocolServer.java   |  4 +-
 .../server/TestSCMSecurityProtocolServer.java   | 60 ++++++++++++++++++++
 .../hadoop/ozone/TestSecureOzoneCluster.java    |  3 +-
 3 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index e810c54..ab29c1e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslator
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.KerberosInfo;
 import org.slf4j.Logger;
@@ -60,6 +60,8 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
     rpcAddress = HddsServerUtil
         .getScmSecurityInetAddress(conf);
     // SCM security service RPC service.
+    RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
+        ProtobufRpcEngine.class);
     BlockingService secureProtoPbService =
         SCMSecurityProtocolProtos.SCMSecurityProtocolService
             .newReflectiveBlockingService(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
new file mode 100644
index 0000000..8e7d84c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Test class for {@link SCMSecurityProtocolServer}.
+ * */
+public class TestSCMSecurityProtocolServer {
+  private SCMSecurityProtocolServer securityProtocolServer;
+  private OzoneConfiguration config;
+
+  @Rule
+  public Timeout timeout = new Timeout(1000 * 20);
+
+  @Before
+  public void setUp() throws Exception {
+    config = new OzoneConfiguration();
+    securityProtocolServer = new SCMSecurityProtocolServer(config, null);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (securityProtocolServer != null) {
+      securityProtocolServer.stop();
+      securityProtocolServer = null;
+    }
+    config = null;
+  }
+
+  @Test
+  public void testStart() {
+    securityProtocolServer.start();
+  }
+
+  @Test
+  public void testStop() {
+    securityProtocolServer.stop();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5791de9..904e597 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -296,7 +296,8 @@ public final class TestSecureOzoneCluster {
     } catch (Exception ex) {
       // Expects timeout failure from scmClient in KSM but KSM user login via
       // kerberos should succeed
-      Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+      Assert.assertTrue(
+          logs.getOutput().contains("Ozone Manager login successful."));
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[32/50] [abbrv] hadoop git commit: HDDS-588. SelfSignedCertificate#generateCertificate should sign the certificate the configured security provider. Contributed by Xiaoyu Yao.

Posted by xy...@apache.org.
HDDS-588. SelfSignedCertificate#generateCertificate should sign the certificate the configured security provider. Contributed by Xiaoyu Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/602aa807
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/602aa807
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/602aa807

Branch: refs/heads/HDDS-4
Commit: 602aa807483464ed3b601ace2a6c379719dff653
Parents: 812c07e
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 9 00:28:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800

----------------------------------------------------------------------
 .../hdds/security/x509/certificates/SelfSignedCertificate.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/602aa807/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index fef7ac3..f221246 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -103,8 +103,8 @@ public final class SelfSignedCertificate {
 
 
     ContentSigner contentSigner =
-        new JcaContentSignerBuilder(
-            config.getSignatureAlgo()).build(key.getPrivate());
+        new JcaContentSignerBuilder(config.getSignatureAlgo())
+            .setProvider(config.getProvider()).build(key.getPrivate());
 
     // Please note: Since this is a root certificate we use "ONE" as the
     // serial number. Also note that skip enforcing locale or UTC. We are


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org