You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/11/29 20:44:17 UTC
[01/50] [abbrv] hadoop git commit: YARN-9061. Improve the GPU/FPGA
module log message of container-executor. (Zhankun Tang via wangda) [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDDS-4 278d4b9b7 -> 187bbbe68 (forced update)
YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda)
Change-Id: Iece9b47438357077a53984a820d4d6423f480518
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed87567
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed87567
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed87567
Branch: refs/heads/HDDS-4
Commit: 9ed87567ad0f1c26a263ce6d8fba56d066260c5c
Parents: 579ef4b
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:31:31 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:31:31 2018 -0800
----------------------------------------------------------------------
.../native/container-executor/impl/modules/fpga/fpga-module.c | 5 +++--
.../native/container-executor/impl/modules/gpu/gpu-module.c | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
index c1a2f83..e947d7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
@@ -141,7 +141,7 @@ void reload_fpga_configuration() {
/*
* Format of FPGA request commandline:
*
- * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y
+ * c-e --module-fpga --excluded_fpgas 0,1,3 --container_id container_x_y
*/
int handle_fpga_request(update_cgroups_parameters_function func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_fpga_request(update_cgroups_parameters_function func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded-fpgas is not specified, skip cgroups call.\n");
goto cleanup;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index 1a1b164..7522338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -141,7 +141,7 @@ void reload_gpu_configuration() {
/*
* Format of GPU request commandline:
*
- * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y
+ * c-e --module-gpu --excluded_gpus 0,1,3 --container_id container_x_y
*/
int handle_gpu_request(update_cgroups_parameters_func func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_gpu_request(update_cgroups_parameters_func func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded_gpus is not specified, skip cgroups call.\n");
goto cleanup;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDDS-804. Block token: Add secret
token manager. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-804. Block token: Add secret token manager. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/187bbbe6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/187bbbe6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/187bbbe6
Branch: refs/heads/HDDS-4
Commit: 187bbbe68cc87729f327e5ed614474d1269b8d85
Parents: 87f51d2
Author: Ajay Kumar <aj...@apache.org>
Authored: Thu Nov 29 08:00:41 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:58:55 2018 -0800
----------------------------------------------------------------------
.../hdds/security/x509/SecurityConfig.java | 9 +
.../security/OzoneBlockTokenSecretManager.java | 191 +++++++
.../OzoneDelegationTokenSecretManager.java | 455 +++++++++++++++++
.../ozone/security/OzoneSecretManager.java | 498 ++++---------------
.../TestOzoneBlockTokenSecretManager.java | 146 ++++++
.../TestOzoneDelegationTokenSecretManager.java | 218 ++++++++
.../ozone/security/TestOzoneSecretManager.java | 216 --------
.../apache/hadoop/ozone/om/OzoneManager.java | 23 +-
.../security/TestOzoneManagerBlockToken.java | 251 ++++++++++
9 files changed, 1371 insertions(+), 636 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index ee20a21..b38ee7c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdds.security.x509;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +78,7 @@ public class SecurityConfig {
private final Duration certDuration;
private final String x509SignatureAlgo;
private final Boolean grpcBlockTokenEnabled;
+ private final int getMaxKeyLength;
private final String certificateDir;
private final String certificateFileName;
@@ -88,6 +90,9 @@ public class SecurityConfig {
public SecurityConfig(Configuration configuration) {
Preconditions.checkNotNull(configuration, "Configuration cannot be null");
this.configuration = configuration;
+ this.getMaxKeyLength = configuration.getInt(
+ OzoneConfigKeys.OZONE_MAX_KEY_LEN,
+ OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
HDDS_DEFAULT_KEY_ALGORITHM);
@@ -289,4 +294,8 @@ public class SecurityConfig {
throw new SecurityException("Unknown security provider:" + provider);
}
}
+
+ public int getMaxKeyLength() {
+ return this.getMaxKeyLength;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
new file mode 100644
index 0000000..3b833cb
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+/**
+ * SecretManager for Ozone Master block tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneBlockTokenSecretManager extends
+ OzoneSecretManager<OzoneBlockTokenIdentifier> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneBlockTokenSecretManager.class);;
+ // Will be set by grpc clients for individual datanodes.
+ static final Text SERVICE = new Text("HDDS_SERVICE");
+ private final String omCertSerialId;
+
+ /**
+ * Create a secret manager.
+ *
+ * @param conf
+ * @param blockTokenExpirytime token expiry time for expired tokens in
+ * milliseconds
+ */
+ public OzoneBlockTokenSecretManager(OzoneConfiguration conf,
+ long blockTokenExpirytime, String omCertSerialId) {
+ super(conf, blockTokenExpirytime, blockTokenExpirytime, SERVICE, LOG);
+ this.omCertSerialId = omCertSerialId;
+ }
+
+ @Override
+ public OzoneBlockTokenIdentifier createIdentifier() {
+ throw new SecurityException("Ozone block token can't be created "
+ + "without owner and access mode information.");
+ }
+
+ public OzoneBlockTokenIdentifier createIdentifier(String owner,
+ String blockId, EnumSet<AccessModeProto> modes, long maxLength) {
+ return new OzoneBlockTokenIdentifier(owner, blockId, modes,
+ getTokenExpiryTime(), omCertSerialId, maxLength);
+ }
+
+ /**
+ * Generate an block token for specified user, blockId.
+ *
+ * @param user
+ * @param blockId
+ * @param modes
+ * @param maxLength
+ * @return token
+ */
+ public Token<OzoneBlockTokenIdentifier> generateToken(String user,
+ String blockId, EnumSet<AccessModeProto> modes, long maxLength) {
+ OzoneBlockTokenIdentifier tokenIdentifier = createIdentifier(user,
+ blockId, modes, maxLength);
+ if (LOG.isTraceEnabled()) {
+ long expiryTime = tokenIdentifier.getExpiryDate();
+ String tokenId = tokenIdentifier.toString();
+ LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+ expiryTime, tokenId);
+ }
+ return new Token<>(tokenIdentifier.getBytes(),
+ createPassword(tokenIdentifier), tokenIdentifier.getKind(), SERVICE);
+ }
+
+ /**
+ * Generate an block token for current user.
+ *
+ * @param blockId
+ * @param modes
+ * @return token
+ */
+ public Token<OzoneBlockTokenIdentifier> generateToken(String blockId,
+ EnumSet<AccessModeProto> modes, long maxLength) throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String userID = (ugi == null ? null : ugi.getShortUserName());
+ return generateToken(userID, blockId, modes, maxLength);
+ }
+
+ @Override
+ public byte[] retrievePassword(OzoneBlockTokenIdentifier identifier)
+ throws InvalidToken {
+ validateToken(identifier);
+ return createPassword(identifier);
+ }
+
+ @Override
+ public long renewToken(Token<OzoneBlockTokenIdentifier> token,
+ String renewer) throws IOException {
+ throw new UnsupportedOperationException("Renew token operation is not " +
+ "supported for ozone block tokens.");
+ }
+
+ @Override
+ public OzoneBlockTokenIdentifier cancelToken(Token<OzoneBlockTokenIdentifier>
+ token, String canceller) throws IOException {
+ throw new UnsupportedOperationException("Cancel token operation is not " +
+ "supported for ozone block tokens.");
+ }
+
+ /**
+ * Find the OzoneBlockTokenInfo for the given token id, and verify that if the
+ * token is not expired.
+ */
+ public boolean validateToken(OzoneBlockTokenIdentifier identifier)
+ throws InvalidToken {
+ long now = Time.now();
+ if (identifier.getExpiryDate() < now) {
+ throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+ "expired, current time: " + Time.formatTime(now) +
+ " expiry time: " + identifier.getExpiryDate());
+ }
+
+ if (!verifySignature(identifier, createPassword(identifier))) {
+ throw new InvalidToken("Tampared/Inavalid token.");
+ }
+ return true;
+ }
+
+ /**
+ * Should be called before this object is used.
+ */
+ @Override
+ public synchronized void start(KeyPair keyPair) throws IOException {
+ super.start(keyPair);
+ removeExpiredKeys();
+ }
+
+ /**
+ * Returns expiry time by adding configured expiry time with current time.
+ *
+ * @return Expiry time.
+ */
+ private long getTokenExpiryTime() {
+ return Time.now() + getTokenRenewInterval();
+ }
+
+ /**
+ * Should be called before this object is used.
+ */
+ @Override
+ public synchronized void stop() throws IOException {
+ super.stop();
+ }
+
+ private synchronized void removeExpiredKeys() {
+ // TODO: handle roll private key/certificate
+ long now = Time.now();
+ for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+ .iterator(); it.hasNext();) {
+ Map.Entry<Integer, OzoneSecretKey> e = it.next();
+ OzoneSecretKey key = e.getValue();
+ if (key.getExpiryDate() < now) {
+ it.remove();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
new file mode 100644
index 0000000..1b9414b
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
@@ -0,0 +1,455 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * SecretManager for Ozone Master. Responsible for signing identifiers with
+ * private key,
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneDelegationTokenSecretManager<T extends OzoneTokenIdentifier>
+ extends OzoneSecretManager<T> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneDelegationTokenSecretManager.class);
+ private final Map<T, TokenInfo> currentTokens;
+ private final OzoneSecretStore store;
+ private Thread tokenRemoverThread;
+ private final long tokenRemoverScanInterval;
+ /**
+ * If the delegation token update thread holds this lock, it will not get
+ * interrupted.
+ */
+ private Object noInterruptsLock = new Object();
+
+ /**
+ * Create a secret manager.
+ *
+ * @param conf configuration.
+ * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
+ * milliseconds
+ * @param tokenRenewInterval how often the tokens must be renewed in
+ * milliseconds
+ * @param dtRemoverScanInterval how often the tokens are scanned for expired
+ * tokens in milliseconds
+ */
+ public OzoneDelegationTokenSecretManager(OzoneConfiguration conf,
+ long tokenMaxLifetime, long tokenRenewInterval,
+ long dtRemoverScanInterval, Text service) throws IOException {
+ super(conf, tokenMaxLifetime, tokenRenewInterval, service, LOG);
+ currentTokens = new ConcurrentHashMap();
+ this.tokenRemoverScanInterval = dtRemoverScanInterval;
+ this.store = new OzoneSecretStore(conf);
+ loadTokenSecretState(store.loadState());
+ }
+
+ @Override
+ public T createIdentifier() {
+ return (T) T.newInstance();
+ }
+
+ /**
+ * Create new Identifier with given,owner,renwer and realUser.
+ *
+ * @return T
+ */
+ public T createIdentifier(Text owner, Text renewer, Text realUser) {
+ return (T) T.newInstance(owner, renewer, realUser);
+ }
+
+ /**
+ * Returns {@link Token} for given identifier.
+ *
+ * @param owner
+ * @param renewer
+ * @param realUser
+ * @return Token
+ * @throws IOException to allow future exceptions to be added without breaking
+ * compatibility
+ */
+ public Token<T> createToken(Text owner, Text renewer, Text realUser)
+ throws IOException {
+ T identifier = createIdentifier(owner, renewer, realUser);
+ updateIdentifierDetails(identifier);
+
+ byte[] password = createPassword(identifier.getBytes(),
+ getCurrentKey().getPrivateKey());
+ addToTokenStore(identifier, password);
+ Token<T> token = new Token<>(identifier.getBytes(), password,
+ identifier.getKind(), getService());
+ if (LOG.isTraceEnabled()) {
+ long expiryTime = identifier.getIssueDate() + getTokenRenewInterval();
+ String tokenId = identifier.toStringStable();
+ LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+ expiryTime, tokenId);
+ }
+ return token;
+ }
+
+ /**
+ * Stores given identifier in token store.
+ *
+ * @param identifier
+ * @param password
+ * @throws IOException
+ */
+ private void addToTokenStore(T identifier, byte[] password)
+ throws IOException {
+ TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
+ + getTokenRenewInterval(), password, identifier.getTrackingId());
+ currentTokens.put(identifier, tokenInfo);
+ store.storeToken(identifier, tokenInfo.getRenewDate());
+ }
+
+ /**
+ * Updates issue date, master key id and sequence number for identifier.
+ *
+ * @param identifier the identifier to validate
+ */
+ private void updateIdentifierDetails(T identifier) {
+ int sequenceNum;
+ long now = Time.monotonicNow();
+ sequenceNum = incrementDelegationTokenSeqNum();
+ identifier.setIssueDate(now);
+ identifier.setMasterKeyId(getCurrentKey().getKeyId());
+ identifier.setSequenceNumber(sequenceNum);
+ identifier.setMaxDate(Time.monotonicNow() + getTokenMaxLifetime());
+ }
+
+ /**
+ * Renew a delegation token.
+ *
+ * @param token the token to renew
+ * @param renewer the full principal name of the user doing the renewal
+ * @return the new expiration time
+ * @throws InvalidToken if the token is invalid
+ * @throws AccessControlException if the user can't renew token
+ */
+ @Override
+ public synchronized long renewToken(Token<T> token, String renewer)
+ throws IOException {
+ ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ T id = (T) T.readProtoBuf(in);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
+ formatTokenId(id), currentTokens.size());
+ }
+
+ long now = Time.monotonicNow();
+ if (id.getMaxDate() < now) {
+ throw new InvalidToken(renewer + " tried to renew an expired token "
+ + formatTokenId(id) + " max expiration date: "
+ + Time.formatTime(id.getMaxDate())
+ + " currentTime: " + Time.formatTime(now));
+ }
+ validateToken(id);
+ if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
+ throw new AccessControlException(renewer +
+ " tried to renew a token " + formatTokenId(id)
+ + " without a renewer");
+ }
+ if (!id.getRenewer().toString().equals(renewer)) {
+ throw new AccessControlException(renewer
+ + " tries to renew a token " + formatTokenId(id)
+ + " with non-matching renewer " + id.getRenewer());
+ }
+ OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
+ if (key == null) {
+ throw new InvalidToken("Unable to find master key for keyId="
+ + id.getMasterKeyId()
+ + " from cache. Failed to renew an unexpired token "
+ + formatTokenId(id) + " with sequenceNumber="
+ + id.getSequenceNumber());
+ }
+ byte[] password = createPassword(token.getIdentifier(),
+ key.getPrivateKey());
+
+ long renewTime = Math.min(id.getMaxDate(), now + getTokenRenewInterval());
+ try {
+ addToTokenStore(id, password);
+ } catch (IOException e) {
+ LOG.error("Unable to update token " + id.getSequenceNumber(), e);
+ }
+ return renewTime;
+ }
+
+ /**
+ * Cancel a token by removing it from store and cache.
+ *
+ * @return Identifier of the canceled token
+ * @throws InvalidToken for invalid token
+ * @throws AccessControlException if the user isn't allowed to cancel
+ */
+ public T cancelToken(Token<T> token, String canceller) throws IOException {
+ T id = (T) T.readProtoBuf(token.getIdentifier());
+ LOG.debug("Token cancellation requested for identifier: {}",
+ formatTokenId(id));
+
+ if (id.getUser() == null) {
+ throw new InvalidToken("Token with no owner " + formatTokenId(id));
+ }
+ String owner = id.getUser().getUserName();
+ Text renewer = id.getRenewer();
+ HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
+ String cancelerShortName = cancelerKrbName.getShortName();
+ if (!canceller.equals(owner)
+ && (renewer == null || renewer.toString().isEmpty()
+ || !cancelerShortName
+ .equals(renewer.toString()))) {
+ throw new AccessControlException(canceller
+ + " is not authorized to cancel the token " + formatTokenId(id));
+ }
+ try {
+ store.removeToken(id);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
+ }
+ TokenInfo info = currentTokens.remove(id);
+ if (info == null) {
+ throw new InvalidToken("Token not found " + formatTokenId(id));
+ }
+ return id;
+ }
+
+ @Override
+ public byte[] retrievePassword(T identifier) throws InvalidToken {
+ return validateToken(identifier).getPassword();
+ }
+
+ /**
+ * Checks if TokenInfo for the given identifier exists in database and if the
+ * token is expired.
+ */
+ public TokenInfo validateToken(T identifier) throws InvalidToken {
+ TokenInfo info = currentTokens.get(identifier);
+ if (info == null) {
+ throw new InvalidToken("token " + formatTokenId(identifier)
+ + " can't be found in cache");
+ }
+ long now = Time.monotonicNow();
+ if (info.getRenewDate() < now) {
+ throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+ "expired, current time: " + Time.formatTime(now) +
+ " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+ }
+ if (!verifySignature(identifier, info.getPassword())) {
+ throw new InvalidToken("Tampared/Inavalid token.");
+ }
+ return info;
+ }
+
+ // TODO: handle roll private key/certificate
+ private synchronized void removeExpiredKeys() {
+ long now = Time.monotonicNow();
+ for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+ .iterator(); it.hasNext();) {
+ Map.Entry<Integer, OzoneSecretKey> e = it.next();
+ OzoneSecretKey key = e.getValue();
+ if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
+ if (!key.equals(getCurrentKey())) {
+ it.remove();
+ try {
+ store.removeTokenMasterKey(key);
+ } catch (IOException ex) {
+ LOG.error("Unable to remove master key " + key.getKeyId(), ex);
+ }
+ }
+ }
+ }
+ }
+
+ private void loadTokenSecretState(OzoneManagerSecretState<T> state)
+ throws IOException {
+ LOG.info("Loading token state into token manager.");
+ for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
+ allKeys.putIfAbsent(key.getKeyId(), key);
+ incrementCurrentKeyId();
+ }
+ for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
+ addPersistedDelegationToken(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void addPersistedDelegationToken(
+ T identifier, long renewDate)
+ throws IOException {
+ if (isRunning()) {
+ // a safety check
+ throw new IOException(
+ "Can't add persisted delegation token to a running SecretManager.");
+ }
+ int keyId = identifier.getMasterKeyId();
+ OzoneSecretKey dKey = allKeys.get(keyId);
+ if (dKey == null) {
+ LOG.warn("No KEY found for persisted identifier "
+ + formatTokenId(identifier));
+ return;
+ }
+
+ PrivateKey privateKey = dKey.getPrivateKey();
+ byte[] password = createPassword(identifier.getBytes(), privateKey);
+ if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
+ setDelegationTokenSeqNum(identifier.getSequenceNumber());
+ }
+ if (currentTokens.get(identifier) == null) {
+ currentTokens.put(identifier, new TokenInfo(renewDate,
+ password, identifier.getTrackingId()));
+ } else {
+ throw new IOException("Same delegation token being added twice: "
+ + formatTokenId(identifier));
+ }
+ }
+
+ /**
+ * Should be called before this object is used.
+ */
+ @Override
+ public synchronized void start(KeyPair keyPair) throws IOException {
+ super.start(keyPair);
+ storeKey(getCurrentKey());
+ removeExpiredKeys();
+ tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
+ tokenRemoverThread.start();
+ }
+
+ private void storeKey(OzoneSecretKey key) throws IOException {
+ store.storeTokenMasterKey(key);
+ if (!allKeys.containsKey(key.getKeyId())) {
+ allKeys.put(key.getKeyId(), key);
+ }
+ }
+
+ public void stopThreads() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping expired delegation token remover thread");
+ }
+ setIsRunning(false);
+
+ if (tokenRemoverThread != null) {
+ synchronized (noInterruptsLock) {
+ tokenRemoverThread.interrupt();
+ }
+ try {
+ tokenRemoverThread.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(
+ "Unable to join on token removal thread", e);
+ }
+ }
+ }
+
+ /**
+ * Stops the OzoneDelegationTokenSecretManager.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void stop() throws IOException {
+ super.stop();
+ stopThreads();
+ if (this.store != null) {
+ this.store.close();
+ }
+ }
+
+ /**
+ * Remove expired delegation tokens from cache and persisted store.
+ */
+ private void removeExpiredToken() {
+ long now = Time.monotonicNow();
+ synchronized (this) {
+ Iterator<Map.Entry<T,
+ TokenInfo>> i = currentTokens.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry<T,
+ TokenInfo> entry = i.next();
+ long renewDate = entry.getValue().getRenewDate();
+ if (renewDate < now) {
+ i.remove();
+ try {
+ store.removeToken(entry.getKey());
+ } catch (IOException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Failed to remove expired token {}", entry.getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private class ExpiredTokenRemover extends Thread {
+ private long lastTokenCacheCleanup;
+
+ @Override
+ public void run() {
+ LOG.info("Starting expired delegation token remover thread, "
+ + "tokenRemoverScanInterval=" + getTokenRemoverScanInterval()
+ / (60 * 1000) + " min(s)");
+ try {
+ while (isRunning()) {
+ long now = Time.monotonicNow();
+ if (lastTokenCacheCleanup + getTokenRemoverScanInterval()
+ < now) {
+ removeExpiredToken();
+ lastTokenCacheCleanup = now;
+ }
+ try {
+ Thread.sleep(Math.min(5000,
+ getTokenRemoverScanInterval())); // 5 seconds
+ } catch (InterruptedException ie) {
+ LOG.error("ExpiredTokenRemover received " + ie);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("ExpiredTokenRemover thread received unexpected exception",
+ t);
+ Runtime.getRuntime().exit(-1);
+ }
+ }
+ }
+
+ public long getTokenRemoverScanInterval() {
+ return tokenRemoverScanInterval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
index 0c84404..01ef8bb 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
@@ -18,8 +18,17 @@
package org.apache.hadoop.ozone.security;
import com.google.common.base.Preconditions;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.slf4j.Logger;
+
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.KeyPair;
@@ -27,25 +36,9 @@ import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.Signature;
import java.security.SignatureException;
-import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
-import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.HadoopKerberosName;
-import org.apache.hadoop.security.token.SecretManager;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* SecretManager for Ozone Master. Responsible for signing identifiers with
@@ -53,33 +46,23 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class OzoneSecretManager<T extends OzoneTokenIdentifier>
+public abstract class OzoneSecretManager<T extends TokenIdentifier>
extends SecretManager<T> {
- private static final Logger LOG = LoggerFactory
- .getLogger(OzoneSecretManager.class);
+ private final Logger logger;
/**
* The name of the Private/Public Key based hashing algorithm.
*/
- private static final String DEFAULT_SIGNATURE_ALGORITHM = "SHA256withRSA";
+ private final SecurityConfig securityConfig;
private final long tokenMaxLifetime;
private final long tokenRenewInterval;
- private final long tokenRemoverScanInterval;
private final Text service;
- private final Map<Integer, OzoneSecretKey> allKeys;
- private final Map<T, TokenInfo> currentTokens;
- private final OzoneSecretStore store;
- private Thread tokenRemoverThread;
private volatile boolean running;
- private AtomicInteger tokenSequenceNumber;
private OzoneSecretKey currentKey;
- private AtomicInteger currentKeyId;
- /**
- * If the delegation token update thread holds this lock, it will not get
- * interrupted.
- */
- private Object noInterruptsLock = new Object();
private int maxKeyLength;
+ private AtomicInteger currentKeyId;
+ private AtomicInteger tokenSequenceNumber;
+ protected final Map<Integer, OzoneSecretKey> allKeys;
/**
* Create a secret manager.
@@ -89,100 +72,21 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
* milliseconds
* @param tokenRenewInterval how often the tokens must be renewed in
* milliseconds
- * @param dtRemoverScanInterval how often the tokens are scanned for expired
- * tokens in milliseconds
+ * @param service name of service
*/
public OzoneSecretManager(OzoneConfiguration conf, long tokenMaxLifetime,
- long tokenRenewInterval, long dtRemoverScanInterval, Text service)
- throws IOException {
+ long tokenRenewInterval, Text service, Logger logger) {
+ this.securityConfig = new SecurityConfig(conf);
this.tokenMaxLifetime = tokenMaxLifetime;
this.tokenRenewInterval = tokenRenewInterval;
- this.tokenRemoverScanInterval = dtRemoverScanInterval;
-
- currentTokens = new ConcurrentHashMap();
- allKeys = new ConcurrentHashMap<>();
currentKeyId = new AtomicInteger();
tokenSequenceNumber = new AtomicInteger();
- this.store = new OzoneSecretStore(conf);
- loadTokenSecretState(store.loadState());
+ allKeys = new ConcurrentHashMap<>();
this.service = service;
- this.maxKeyLength = conf.getInt(OzoneConfigKeys.OZONE_MAX_KEY_LEN,
- OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
- }
-
- @Override
- public T createIdentifier() {
- return (T) T.newInstance();
+ this.maxKeyLength = securityConfig.getMaxKeyLength();
+ this.logger = logger;
}
- /**
- * Create new Identifier with given,owner,renwer and realUser.
- *
- * @return T
- */
- public T createIdentifier(Text owner, Text renewer, Text realUser) {
- return (T) T.newInstance(owner, renewer, realUser);
- }
-
- /**
- * Returns {@link Token} for given identifier.
- *
- * @param owner
- * @param renewer
- * @param realUser
- * @return Token
- * @throws IOException to allow future exceptions to be added without breaking
- * compatibility
- */
- public Token<T> createToken(Text owner, Text renewer, Text realUser)
- throws IOException {
- T identifier = createIdentifier(owner, renewer, realUser);
- updateIdentifierDetails(identifier);
-
- byte[] password = createPassword(identifier.getBytes(),
- currentKey.getPrivateKey());
- addToTokenStore(identifier, password);
- Token<T> token = new Token<>(identifier.getBytes(), password,
- identifier.getKind(), service);
- if (LOG.isTraceEnabled()) {
- long expiryTime = identifier.getIssueDate() + tokenRenewInterval;
- String tokenId = identifier.toStringStable();
- LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
- expiryTime, tokenId);
- }
-
- return token;
- }
-
- /**
- * Stores given identifier in token store.
- *
- * @param identifier
- * @param password
- * @throws IOException
- */
- private void addToTokenStore(T identifier, byte[] password)
- throws IOException {
- TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
- + tokenRenewInterval, password, identifier.getTrackingId());
- currentTokens.put(identifier, tokenInfo);
- store.storeToken(identifier, tokenInfo.getRenewDate());
- }
-
- /**
- * Updates issue date, master key id and sequence number for identifier.
- *
- * @param identifier the identifier to validate
- */
- private void updateIdentifierDetails(T identifier) {
- int sequenceNum;
- long now = Time.monotonicNow();
- sequenceNum = incrementDelegationTokenSeqNum();
- identifier.setIssueDate(now);
- identifier.setMasterKeyId(currentKey.getKeyId());
- identifier.setSequenceNumber(sequenceNum);
- identifier.setMaxDate(Time.monotonicNow() + tokenMaxLifetime);
- }
/**
* Compute HMAC of the identifier using the private key and return the output
@@ -196,7 +100,7 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
throws OzoneSecurityException {
try {
Signature rsaSignature = Signature.getInstance(
- DEFAULT_SIGNATURE_ALGORITHM);
+ getDefaultSignatureAlgorithm());
rsaSignature.initSign(privateKey);
rsaSignature.update(identifier);
return rsaSignature.sign();
@@ -210,22 +114,31 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
@Override
public byte[] createPassword(T identifier) {
- LOG.debug("Creating password for identifier: {}, currentKey: {}",
+ logger.debug("Creating password for identifier: {}, currentKey: {}",
formatTokenId(identifier), currentKey.getKeyId());
byte[] password = null;
try {
password = createPassword(identifier.getBytes(),
currentKey.getPrivateKey());
} catch (IOException ioe) {
- LOG.error("Could not store token {}!!", formatTokenId(identifier),
+ logger.error("Could not store token {}!!", formatTokenId(identifier),
ioe);
}
return password;
}
+ /**
+ * Default implementation for Ozone. Verifies if hash in token is legit.
+ * */
@Override
public byte[] retrievePassword(T identifier) throws InvalidToken {
- return checkToken(identifier).getPassword();
+ byte[] password = createPassword(identifier);
+ // TODO: Revisit this when key/certificate rotation is implemented.
+ // i.e Try all valid keys instead of current key only.
+ if (!verifySignature(identifier, password)) {
+ throw new InvalidToken("Tampared/Inavalid token.");
+ }
+ return password;
}
/**
@@ -237,52 +150,8 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
* @throws InvalidToken if the token is invalid
* @throws AccessControlException if the user can't renew token
*/
- public synchronized long renewToken(Token<T> token, String renewer)
- throws IOException {
- ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
- DataInputStream in = new DataInputStream(buf);
- T id = (T) T.readProtoBuf(in);
- LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
- formatTokenId(id), currentTokens.size());
-
- long now = Time.monotonicNow();
- if (id.getMaxDate() < now) {
- throw new InvalidToken(renewer + " tried to renew an expired token "
- + formatTokenId(id) + " max expiration date: "
- + Time.formatTime(id.getMaxDate())
- + " currentTime: " + Time.formatTime(now));
- }
- checkToken(id);
- if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
- throw new AccessControlException(renewer +
- " tried to renew a token " + formatTokenId(id)
- + " without a renewer");
- }
- if (!id.getRenewer().toString().equals(renewer)) {
- throw new AccessControlException(renewer
- + " tries to renew a token " + formatTokenId(id)
- + " with non-matching renewer " + id.getRenewer());
- }
- OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
- if (key == null) {
- throw new InvalidToken("Unable to find master key for keyId="
- + id.getMasterKeyId()
- + " from cache. Failed to renew an unexpired token "
- + formatTokenId(id) + " with sequenceNumber="
- + id.getSequenceNumber());
- }
- byte[] password = createPassword(token.getIdentifier(),
- key.getPrivateKey());
-
- long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
- try {
- addToTokenStore(id, password);
- } catch (IOException e) {
- LOG.error("Unable to update token " + id.getSequenceNumber(), e);
- }
- return renewTime;
- }
-
+ public abstract long renewToken(Token<T> token, String renewer)
+ throws IOException;
/**
* Cancel a token by removing it from store and cache.
*
@@ -290,44 +159,8 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
* @throws InvalidToken for invalid token
* @throws AccessControlException if the user isn't allowed to cancel
*/
- public T cancelToken(Token<T> token, String canceller) throws IOException {
- T id = (T) T.readProtoBuf(token.getIdentifier());
- LOG.debug("Token cancellation requested for identifier: {}",
- formatTokenId(id));
-
- if (id.getUser() == null) {
- throw new InvalidToken("Token with no owner " + formatTokenId(id));
- }
- String owner = id.getUser().getUserName();
- Text renewer = id.getRenewer();
- HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
- String cancelerShortName = cancelerKrbName.getShortName();
- if (!canceller.equals(owner)
- && (renewer == null || renewer.toString().isEmpty()
- || !cancelerShortName
- .equals(renewer.toString()))) {
- throw new AccessControlException(canceller
- + " is not authorized to cancel the token " + formatTokenId(id));
- }
- try {
- store.removeToken(id);
- } catch (IOException e) {
- LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
- }
- TokenInfo info = currentTokens.remove(id);
- if (info == null) {
- throw new InvalidToken("Token not found " + formatTokenId(id));
- }
- return id;
- }
-
- public int getCurrentKeyId() {
- return currentKeyId.get();
- }
-
- public void setCurrentKeyId(int keyId) {
- currentKeyId.set(keyId);
- }
+ public abstract T cancelToken(Token<T> token, String canceller)
+ throws IOException;
public int incrementCurrentKeyId() {
return currentKeyId.incrementAndGet();
@@ -346,14 +179,31 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
}
/**
- * Validates if given token is valid.
+ * Update the current master key. This is called once by start method before
+ * tokenRemoverThread is created,
+ */
+ private OzoneSecretKey updateCurrentKey(KeyPair keyPair) throws IOException {
+ logger.info("Updating the current master key for generating tokens");
+
+ // TODO: fix me based on the certificate expire time to set the key
+ // expire time.
+ int newCurrentId = incrementCurrentKeyId();
+ OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
+ keyPair, maxKeyLength);
+ currentKey = newKey;
+ return currentKey;
+ }
+
+ /**
+ * Validates if given hash is valid.
*
* @param identifier
* @param password
*/
- private boolean validateToken(T identifier, byte[] password) {
+ public boolean verifySignature(T identifier, byte[] password) {
try {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ Signature rsaSignature =
+ Signature.getInstance(getDefaultSignatureAlgorithm());
rsaSignature.initVerify(currentKey.getPublicKey());
rsaSignature.update(identifier.getBytes());
return rsaSignature.verify(password);
@@ -363,179 +213,45 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
}
}
- /**
- * Checks if TokenInfo for the given identifier exists in database and if the
- * token is expired.
- */
- public TokenInfo checkToken(T identifier) throws InvalidToken {
- TokenInfo info = currentTokens.get(identifier);
- if (info == null) {
- throw new InvalidToken("token " + formatTokenId(identifier)
- + " can't be found in cache");
- }
- long now = Time.monotonicNow();
- if (info.getRenewDate() < now) {
- throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
- "expired, current time: " + Time.formatTime(now) +
- " expected renewal time: " + Time.formatTime(info.getRenewDate()));
- }
- if (!validateToken(identifier, info.getPassword())) {
- throw new InvalidToken("Tampared/Inavalid token.");
- }
- return info;
- }
-
- // TODO: handle roll private key/certificate
- private synchronized void removeExpiredKeys() {
- long now = Time.monotonicNow();
- for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
- .iterator(); it.hasNext();) {
- Map.Entry<Integer, OzoneSecretKey> e = it.next();
- OzoneSecretKey key = e.getValue();
- if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
- if (!key.equals(currentKey)) {
- it.remove();
- try {
- store.removeTokenMasterKey(key);
- } catch (IOException ex) {
- LOG.error("Unable to remove master key " + key.getKeyId(), ex);
- }
- }
- }
- }
- }
-
- private void loadTokenSecretState(OzoneManagerSecretState<T> state)
- throws IOException {
- LOG.info("Loading token state into token manager.");
- for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
- allKeys.putIfAbsent(key.getKeyId(), key);
- }
- for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
- addPersistedDelegationToken(entry.getKey(), entry.getValue());
- }
- }
-
- private String formatTokenId(T id) {
+ public String formatTokenId(T id) {
return "(" + id + ")";
}
- private void addPersistedDelegationToken(
- T identifier, long renewDate)
- throws IOException {
- if (running) {
- // a safety check
- throw new IOException(
- "Can't add persisted delegation token to a running SecretManager.");
- }
- int keyId = identifier.getMasterKeyId();
- OzoneSecretKey dKey = allKeys.get(keyId);
- if (dKey == null) {
- LOG.warn("No KEY found for persisted identifier "
- + formatTokenId(identifier));
- return;
- }
-
- PrivateKey privateKey = dKey.getPrivateKey();
- byte[] password = createPassword(identifier.getBytes(), privateKey);
- if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
- setDelegationTokenSeqNum(identifier.getSequenceNumber());
- }
- if (currentTokens.get(identifier) == null) {
- currentTokens.put(identifier, new TokenInfo(renewDate,
- password, identifier.getTrackingId()));
- } else {
- throw new IOException("Same delegation token being added twice: "
- + formatTokenId(identifier));
- }
- }
-
/**
* Should be called before this object is used.
+ *
+ * @param keyPair
+ * @throws IOException
*/
- public void startThreads(KeyPair keyPair) throws IOException {
- Preconditions.checkState(!running);
+ public synchronized void start(KeyPair keyPair) throws IOException {
+ Preconditions.checkState(!isRunning());
updateCurrentKey(keyPair);
- removeExpiredKeys();
- synchronized (this) {
- running = true;
- tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
- tokenRemoverThread.start();
- }
- }
-
- public void stopThreads() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stopping expired delegation token remover thread");
- }
- running = false;
-
- if (tokenRemoverThread != null) {
- synchronized (noInterruptsLock) {
- tokenRemoverThread.interrupt();
- }
- try {
- tokenRemoverThread.join();
- } catch (InterruptedException e) {
- throw new RuntimeException(
- "Unable to join on token removal thread", e);
- }
- }
+ setIsRunning(true);
}
/**
- * Stops the OzoneSecretManager.
+ * Stops the OzoneDelegationTokenSecretManager.
*
* @throws IOException
*/
- public void stop() throws IOException {
- stopThreads();
- if (this.store != null) {
- this.store.close();
- }
+ public synchronized void stop() throws IOException {
+ setIsRunning(false);
}
- /**
- * Update the current master key. This is called once by startThreads before
- * tokenRemoverThread is created,
- */
- private void updateCurrentKey(KeyPair keyPair) throws IOException {
- LOG.info("Updating the current master key for generating tokens");
-
- // TODO: fix me based on the certificate expire time to set the key
- // expire time.
- int newCurrentId = incrementCurrentKeyId();
- OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
- keyPair, maxKeyLength);
+ public String getDefaultSignatureAlgorithm() {
+ return securityConfig.getSignatureAlgo();
+ }
- store.storeTokenMasterKey(newKey);
- if (!allKeys.containsKey(newKey.getKeyId())) {
- allKeys.put(newKey.getKeyId(), newKey);
- }
+ public long getTokenMaxLifetime() {
+ return tokenMaxLifetime;
+ }
- synchronized (this) {
- currentKey = newKey;
- }
+ public long getTokenRenewInterval() {
+ return tokenRenewInterval;
}
- /**
- * Remove expired delegation tokens from cache and persisted store.
- */
- private void removeExpiredToken() throws IOException {
- long now = Time.monotonicNow();
- synchronized (this) {
- Iterator<Map.Entry<T,
- TokenInfo>> i = currentTokens.entrySet().iterator();
- while (i.hasNext()) {
- Map.Entry<T,
- TokenInfo> entry = i.next();
- long renewDate = entry.getValue().getRenewDate();
- if (renewDate < now) {
- i.remove();
- store.removeToken(entry.getKey());
- }
- }
- }
+ public Text getService() {
+ return service;
}
/**
@@ -547,52 +263,20 @@ public class OzoneSecretManager<T extends OzoneTokenIdentifier>
return running;
}
- /**
- * Returns expiry time of a token given its identifier.
- *
- * @param dtId DelegationTokenIdentifier of a token
- * @return Expiry time of the token
- * @throws IOException
- */
- public long getTokenExpiryTime(T dtId)
- throws IOException {
- TokenInfo info = currentTokens.get(dtId);
- if (info != null) {
- return info.getRenewDate();
- } else {
- throw new IOException("No delegation token found for this identifier");
- }
+ public void setIsRunning(boolean val) {
+ running = val;
}
- private class ExpiredTokenRemover extends Thread {
- private long lastTokenCacheCleanup;
+ public OzoneSecretKey getCurrentKey() {
+ return currentKey;
+ }
- @Override
- public void run() {
- LOG.info("Starting expired delegation token remover thread, "
- + "tokenRemoverScanInterval=" + tokenRemoverScanInterval
- / (60 * 1000) + " min(s)");
- try {
- while (running) {
- long now = Time.monotonicNow();
- if (lastTokenCacheCleanup + tokenRemoverScanInterval
- < now) {
- removeExpiredToken();
- lastTokenCacheCleanup = now;
- }
- try {
- Thread.sleep(Math.min(5000,
- tokenRemoverScanInterval)); // 5 seconds
- } catch (InterruptedException ie) {
- LOG.error("ExpiredTokenRemover received " + ie);
- }
- }
- } catch (Throwable t) {
- LOG.error("ExpiredTokenRemover thread received unexpected exception",
- t);
- Runtime.getRuntime().exit(-1);
- }
- }
+ public AtomicInteger getCurrentKeyId() {
+ return currentKeyId;
+ }
+
+ public AtomicInteger getTokenSequenceNumber() {
+ return tokenSequenceNumber;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
new file mode 100644
index 0000000..4692266
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenSecretManager.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.security.KeyPair;
+import java.security.Signature;
+import java.security.cert.X509Certificate;
+import java.util.EnumSet;
+
+/**
+ * Test class for {@link OzoneBlockTokenSecretManager}.
+ */
+public class TestOzoneBlockTokenSecretManager {
+
+ private OzoneBlockTokenSecretManager secretManager;
+ private KeyPair keyPair;
+ private X509Certificate x509Certificate;
+ private long expiryTime;
+ private String omCertSerialId;
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneBlockTokenSecretManager.class.getSimpleName());
+
+
+ @Before
+ public void setUp() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ x509Certificate = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ omCertSerialId = x509Certificate.getSerialNumber().toString();
+ secretManager = new OzoneBlockTokenSecretManager(conf,
+ expiryTime, omCertSerialId);
+ secretManager.start(keyPair);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ secretManager = null;
+ }
+
+ @Test
+ public void testGenerateToken() throws Exception {
+ Token<OzoneBlockTokenIdentifier> token = secretManager.generateToken(
+ "101", EnumSet.allOf(AccessModeProto.class), 100);
+ OzoneBlockTokenIdentifier identifier =
+ OzoneBlockTokenIdentifier.readFieldsProtobuf(new DataInputStream(
+ new ByteArrayInputStream(token.getIdentifier())));
+ // Check basic details.
+ Assert.assertTrue(identifier.getBlockId().equals("101"));
+ Assert.assertTrue(identifier.getAccessModes().equals(EnumSet
+ .allOf(AccessModeProto.class)));
+ Assert.assertTrue(identifier.getOmCertSerialId().equals(omCertSerialId));
+
+ validateHash(token.getPassword(), token.getIdentifier());
+ }
+
+ @Test
+ public void testCreateIdentifierSuccess() throws Exception {
+ OzoneBlockTokenIdentifier btIdentifier = secretManager.createIdentifier(
+ "testUser", "101", EnumSet.allOf(AccessModeProto.class), 100);
+
+ // Check basic details.
+ Assert.assertTrue(btIdentifier.getOwnerId().equals("testUser"));
+ Assert.assertTrue(btIdentifier.getBlockId().equals("101"));
+ Assert.assertTrue(btIdentifier.getAccessModes().equals(EnumSet
+ .allOf(AccessModeProto.class)));
+ Assert.assertTrue(btIdentifier.getOmCertSerialId().equals(omCertSerialId));
+
+ byte[] hash = secretManager.createPassword(btIdentifier);
+ validateHash(hash, btIdentifier.getBytes());
+ }
+
+ /**
+ * Validate hash using public key of KeyPair.
+ * */
+ private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+ Signature rsaSignature =
+ Signature.getInstance(secretManager.getDefaultSignatureAlgorithm());
+ rsaSignature.initVerify(keyPair.getPublic());
+ rsaSignature.update(identifier);
+ Assert.assertTrue(rsaSignature.verify(hash));
+ }
+
+ @Test
+ public void testCreateIdentifierFailure() throws Exception {
+ LambdaTestUtils.intercept(SecurityException.class,
+ "Ozone block token can't be created without owner and access mode "
+ + "information.", () -> {
+ secretManager.createIdentifier();
+ });
+ }
+
+ @Test
+ public void testRenewToken() throws Exception {
+ LambdaTestUtils.intercept(UnsupportedOperationException.class,
+ "Renew token operation is not supported for ozone block" +
+ " tokens.", () -> {
+ secretManager.renewToken(null, null);
+ });
+ }
+
+ @Test
+ public void testCancelToken() throws Exception {
+ LambdaTestUtils.intercept(UnsupportedOperationException.class,
+ "Cancel token operation is not supported for ozone block" +
+ " tokens.", () -> {
+ secretManager.cancelToken(null, null);
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
new file mode 100644
index 0000000..37ad5ce
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.Signature;
+
+/**
+ * Test class for {@link OzoneDelegationTokenSecretManager}.
+ */
+public class TestOzoneDelegationTokenSecretManager {
+
+ private OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+ secretManager;
+ private SecurityConfig securityConfig;
+ private KeyPair keyPair;
+ private long expiryTime;
+ private Text serviceRpcAdd;
+ private OzoneConfiguration conf;
+ private static final String BASEDIR = GenericTestUtils.getTempPath(
+ TestOzoneDelegationTokenSecretManager.class.getSimpleName());
+ private final static Text TEST_USER = new Text("testUser");
+ private long tokenMaxLifetime = 1000 * 20;
+ private long tokenRemoverScanInterval = 1000 * 20;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+ securityConfig = new SecurityConfig(conf);
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+ serviceRpcAdd = new Text("localhost");
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ secretManager.stop();
+ FileUtils.deleteQuietly(new File(BASEDIR));
+ }
+
+ @Test
+ public void testCreateToken() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ OzoneTokenIdentifier identifier =
+ OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
+ // Check basic details.
+ Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
+ Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
+ Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
+
+ validateHash(token.getPassword(), token.getIdentifier());
+ }
+
+ @Test
+ public void testRenewTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(10 * 5);
+ long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
+ Assert.assertTrue(renewalTime > 0);
+ }
+
+ /**
+ * Tests failure for mismatch in renewer.
+ */
+ @Test
+ public void testRenewTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser tries to renew a token", () -> {
+ secretManager.renewToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Tests token renew failure due to max time.
+ */
+ @Test
+ public void testRenewTokenFailureMaxTime() throws Exception {
+ secretManager = createSecretManager(conf, 100,
+ 100, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(101);
+ LambdaTestUtils.intercept(IOException.class,
+ "testUser tried to renew an expired token", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ /**
+ * Tests token renew failure due to renewal time.
+ */
+ @Test
+ public void testRenewTokenFailureRenewalTime() throws Exception {
+ secretManager = createSecretManager(conf, 1000 * 10,
+ 10, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(15);
+ LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ @Test
+ public void testCreateIdentifier() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ OzoneTokenIdentifier identifier = secretManager.createIdentifier();
+ // Check basic details.
+ Assert.assertTrue(identifier.getOwner().equals(new Text("")));
+ Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
+ Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
+ }
+
+ @Test
+ public void testCancelTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ secretManager.cancelToken(token, TEST_USER.toString());
+ }
+
+ @Test
+ public void testCancelTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.start(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser is not authorized to cancel the token", () -> {
+ secretManager.cancelToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Validate hash using public key of KeyPair.
+ */
+ private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+ Signature rsaSignature =
+ Signature.getInstance(securityConfig.getSignatureAlgo(),
+ securityConfig.getProvider());
+ rsaSignature.initVerify(keyPair.getPublic());
+ rsaSignature.update(identifier);
+ Assert.assertTrue(rsaSignature.verify(hash));
+ }
+
+ /**
+ * Create instance of {@link OzoneDelegationTokenSecretManager}.
+ */
+ private OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+ createSecretManager(OzoneConfiguration config, long tokenMaxLife,
+ long expiry, long tokenRemoverScanTime) throws IOException {
+ return new OzoneDelegationTokenSecretManager<>(config, tokenMaxLife,
+ expiry, tokenRemoverScanTime, serviceRpcAdd);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
deleted file mode 100644
index e4a8f2b..0000000
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.security;
-
-import java.io.File;
-import java.io.IOException;
-import java.security.KeyPair;
-import java.security.Signature;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for {@link OzoneSecretManager}.
- */
-public class TestOzoneSecretManager {
-
- private OzoneSecretManager<OzoneTokenIdentifier> secretManager;
- private SecurityConfig securityConfig;
- private KeyPair keyPair;
- private long expiryTime;
- private Text serviceRpcAdd;
- private OzoneConfiguration conf;
- private static final String BASEDIR = GenericTestUtils
- .getTempPath(TestOzoneSecretManager.class.getSimpleName());
- private final static Text TEST_USER = new Text("testUser");
- private long tokenMaxLifetime = 1000 * 20;
- private long tokenRemoverScanInterval = 1000 * 20;
-
- @Before
- public void setUp() throws Exception {
- conf = new OzoneConfiguration();
- conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
- securityConfig = new SecurityConfig(conf);
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- expiryTime = Time.monotonicNow() + 60 * 60 * 24;
- serviceRpcAdd = new Text("localhost");
- }
-
- @After
- public void tearDown() throws IOException {
- secretManager.stop();
- FileUtils.deleteQuietly(new File(BASEDIR));
- }
-
- @Test
- public void testCreateToken() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- OzoneTokenIdentifier identifier =
- OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
- // Check basic details.
- Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
- Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
- Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
-
- validateHash(token.getPassword(), token.getIdentifier());
- }
-
- @Test
- public void testRenewTokenSuccess() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- Thread.sleep(10 * 5);
- long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
- Assert.assertTrue(renewalTime > 0);
- }
-
- /**
- * Tests failure for mismatch in renewer.
- */
- @Test
- public void testRenewTokenFailure() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- LambdaTestUtils.intercept(AccessControlException.class,
- "rougeUser tries to renew a token", () -> {
- secretManager.renewToken(token, "rougeUser");
- });
- }
-
- /**
- * Tests token renew failure due to max time.
- */
- @Test
- public void testRenewTokenFailureMaxTime() throws Exception {
- secretManager = createSecretManager(conf, 100,
- 100, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- Thread.sleep(101);
- LambdaTestUtils.intercept(IOException.class,
- "testUser tried to renew an expired token", () -> {
- secretManager.renewToken(token, TEST_USER.toString());
- });
- }
-
- /**
- * Tests token renew failure due to renewal time.
- */
- @Test
- public void testRenewTokenFailureRenewalTime() throws Exception {
- secretManager = createSecretManager(conf, 1000 * 10,
- 10, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- Thread.sleep(15);
- LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
- secretManager.renewToken(token, TEST_USER.toString());
- });
- }
-
- @Test
- public void testCreateIdentifier() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- OzoneTokenIdentifier identifier = secretManager.createIdentifier();
- // Check basic details.
- Assert.assertTrue(identifier.getOwner().equals(new Text("")));
- Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
- Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
- }
-
- @Test
- public void testCancelTokenSuccess() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- secretManager.cancelToken(token, TEST_USER.toString());
- }
-
- @Test
- public void testCancelTokenFailure() throws Exception {
- secretManager = createSecretManager(conf, tokenMaxLifetime,
- expiryTime, tokenRemoverScanInterval);
- secretManager.startThreads(keyPair);
- Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
- TEST_USER,
- TEST_USER);
- LambdaTestUtils.intercept(AccessControlException.class,
- "rougeUser is not authorized to cancel the token", () -> {
- secretManager.cancelToken(token, "rougeUser");
- });
- }
-
- /**
- * Validate hash using public key of KeyPair.
- */
- private void validateHash(byte[] hash, byte[] identifier) throws Exception {
- Signature rsaSignature =
- Signature.getInstance(securityConfig.getSignatureAlgo(),
- securityConfig.getProvider());
- rsaSignature.initVerify(keyPair.getPublic());
- rsaSignature.update(identifier);
- Assert.assertTrue(rsaSignature.verify(hash));
- }
-
- /**
- * Create instance of {@link OzoneSecretManager}.
- */
- private OzoneSecretManager<OzoneTokenIdentifier> createSecretManager(
- OzoneConfiguration config, long tokenMaxLife, long expiry, long
- tokenRemoverScanTime) throws IOException {
- return new OzoneSecretManager<>(config, tokenMaxLife,
- expiry, tokenRemoverScanTime, serviceRpcAdd);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index c6616aa..1e49779 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -49,12 +49,10 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.security.OzoneSecurityException;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
-import org.apache.hadoop.ozone.security.OzoneSecretManager;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
@@ -79,6 +77,7 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -108,9 +107,7 @@ import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
-import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.ozone.security.OzoneSecurityException.ResultCodes.*;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
@@ -155,8 +152,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
+ StartupOption.HELP.getName() + " ]\n";
private static final String OM_DAEMON = "om";
private static boolean securityEnabled = false;
- private static OzoneSecretManager secretManager;
- // TO DO: For testing purpose only, remove before commiting
+ private static OzoneDelegationTokenSecretManager<OzoneTokenIdentifier>
+ secretManager;
private KeyPair keyPair;
private CertificateClient certClient;
private static boolean testSecureOmFlag = false;
@@ -296,9 +293,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
- private OzoneSecretManager createSecretManager(
- OzoneConfiguration conf)
- throws IOException {
+ private OzoneDelegationTokenSecretManager createSecretManager(
+ OzoneConfiguration conf) throws IOException {
long tokenRemoverScanInterval =
conf.getTimeDuration(OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY,
OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT,
@@ -311,8 +307,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
- return new OzoneSecretManager(conf, tokenMaxLifetime, tokenRenewInterval,
- tokenRemoverScanInterval, omRpcAddressTxt);
+ return new OzoneDelegationTokenSecretManager<>(conf, tokenMaxLifetime,
+ tokenRenewInterval, tokenRemoverScanInterval, omRpcAddressTxt);
}
private void stopSecretManager() throws IOException {
@@ -327,7 +323,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try {
readKeyPair();
LOG.info("Starting OM secret manager");
- secretManager.startThreads(keyPair);
+ secretManager.start(keyPair);
} catch (IOException e) {
// Inability to start secret manager
// can't be recovered from.
@@ -351,7 +347,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
certClient.getPrivateKey(OM_DAEMON));
} catch (Exception e) {
throw new OzoneSecurityException("Error reading private file for "
- + "OzoneManager", e, OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
+ + "OzoneManager", e, OzoneSecurityException
+ .ResultCodes.OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/187bbbe6/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
new file mode 100644
index 0000000..cb7caf3
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneManagerBlockToken {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneManagerBlockToken.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneManagerBlockToken.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static long expiryTime;
+ private static KeyPair keyPair;
+ private static X509Certificate cert;
+ private static final long MAX_LEN = 1000;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ }
+
+ @After
+ public void cleanUp() {
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneBlockTokenIdentifier("", "",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValid = rsaSignature.verify(signature);
+ return isValid;
+ }
+
+ private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneBlockTokenIdentifier generateTestToken() {
+ return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+ RandomStringUtils.randomAlphabetic(5),
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), MAX_LEN);
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate omCert;
+ omCert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ kp, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration / testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), omCert);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration / testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration / testTokenCount);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol
interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
new file mode 100644
index 0000000..02d5e28
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests ozone containers via secure grpc/netty.
+ */
+@RunWith(Parameterized.class)
+public class TestSecureOzoneContainer {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestSecureOzoneContainer.class);
+ /**
+ * Set the timeout for every test.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(300000);
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private OzoneConfiguration conf;
+ private SecurityConfig secConfig;
+ private Boolean requireBlockToken;
+ private Boolean hasBlockToken;
+ private Boolean blockTokeExpired;
+
+
+ public TestSecureOzoneContainer(Boolean requireBlockToken,
+ Boolean hasBlockToken, Boolean blockTokenExpired) {
+ this.requireBlockToken = requireBlockToken;
+ this.hasBlockToken = hasBlockToken;
+ this.blockTokeExpired = blockTokenExpired;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> blockTokenOptions() {
+ return Arrays.asList(new Object[][] {
+ {true, true, false},
+ {true, true, true},
+ {true, false, false},
+ {false, true, false},
+ {false, false, false}});
+ }
+
+ @Before
+ public void setup() throws IOException{
+ conf = new OzoneConfiguration();
+ String ozoneMetaPath =
+ GenericTestUtils.getTempPath("ozoneMeta");
+ conf.set(OZONE_METADATA_DIRS, ozoneMetaPath);
+
+ secConfig = new SecurityConfig(conf);
+
+ }
+
+ @Test
+ public void testCreateOzoneContainer() throws Exception {
+ LOG.info("Test case: requireBlockToken: {} hasBlockToken: {} " +
+ "blockTokenExpired: {}.", requireBlockToken, hasBlockToken,
+ blockTokeExpired);
+ conf.setBoolean(HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+ requireBlockToken);
+
+ long containerID = ContainerTestHelper.getTestContainerID();
+ OzoneContainer container = null;
+ System.out.println(System.getProperties().getProperty("java.library.path"));
+ try {
+ Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
+ conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
+ conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline
+ .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE)
+ .getValue());
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+
+ DatanodeDetails dn = TestUtils.randomDatanodeDetails();
+ container = new OzoneContainer(dn, conf, null);
+ //Setting scmId, as we start manually ozone container.
+ container.getDispatcher().setScmId(UUID.randomUUID().toString());
+ container.start();
+
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+ "user1", new String[] {"usergroup"});
+ long expiryDate = (blockTokeExpired) ?
+ Time.now() - 60 * 60 * 2 : Time.now() + 60 * 60 * 24;
+
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "cid:lud:bcsid",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryDate, "1234", 128L);
+
+ int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
+ if (port == 0) {
+ port = secConfig.getConfiguration().getInt(OzoneConfigKeys
+ .DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ }
+ InetSocketAddress addr =
+ new InetSocketAddress(dn.getIpAddress(), port);
+
+ Token<OzoneBlockTokenIdentifier> token =
+ new Token(tokenId.getBytes(), new byte[2], tokenId.getKind(),
+ SecurityUtil.buildTokenService(addr));
+ if (hasBlockToken) {
+ ugi.addToken(token);
+ }
+
+ ugi.doAs(new PrivilegedAction<Void>() {
+ @Override
+ public Void run() {
+ try {
+ XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
+ client.connect();
+ createContainerForTesting(client, containerID);
+ } catch (Exception e) {
+ if (requireBlockToken && hasBlockToken && !blockTokeExpired) {
+ fail("Client with BlockToken should succeed when block token is" +
+ " required.");
+ }
+ if (requireBlockToken && hasBlockToken && blockTokeExpired) {
+ assertTrue("Receive expected exception",
+ e instanceof SCMSecurityException);
+ }
+ if (requireBlockToken && !hasBlockToken) {
+ assertTrue("Receive expected exception", e instanceof
+ SCMSecurityException);
+ }
+ }
+ return null;
+ }
+ });
+ } finally {
+ if (container != null) {
+ container.stop();
+ }
+ }
+ }
+
+ public static void createContainerForTesting(XceiverClientSpi client,
+ long containerID) throws Exception {
+ // Create container
+ ContainerProtos.ContainerCommandRequestProto request =
+ ContainerTestHelper.getCreateContainerRequest(
+ containerID, client.getPipeline());
+ ContainerProtos.ContainerCommandResponseProto response =
+ client.sendCommand(request);
+ Assert.assertNotNull(response);
+ Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
deleted file mode 100644
index 5909916..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import java.io.File;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test class for OzoneManagerDelegationToken.
- */
-public class TestOzoneBlockTokenIdentifier {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestOzoneBlockTokenIdentifier.class);
- private static final String BASEDIR = GenericTestUtils
- .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
- private static final String KEYSTORES_DIR =
- new File(BASEDIR).getAbsolutePath();
- private static long expiryTime;
- private static KeyPair keyPair;
- private static X509Certificate cert;
-
- @BeforeClass
- public static void setUp() throws Exception {
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- base.mkdirs();
- expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- // Create Ozone Master certificate (SCM CA issued cert) and key store.
- cert = KeyStoreTestUtil
- .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
- }
-
- @After
- public void cleanUp() throws Exception {
- // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
- }
-
- @Test
- public void testSignToken() throws GeneralSecurityException, IOException {
- String keystore = new File(KEYSTORES_DIR, "keystore.jks")
- .getAbsolutePath();
- String truststore = new File(KEYSTORES_DIR, "truststore.jks")
- .getAbsolutePath();
- String trustPassword = "trustPass";
- String keyStorePassword = "keyStorePass";
- String keyPassword = "keyPass";
-
-
- KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
- "OzoneMaster", keyPair.getPrivate(), cert);
-
- // Create trust store and put the certificate in the trust store
- Map<String, X509Certificate> certs = Collections.singletonMap("server",
- cert);
- KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
- // Sign the OzoneMaster Token with Ozone Master private key
- PrivateKey privateKey = keyPair.getPrivate();
- OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
- "testUser", "84940",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
- // Verify a valid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
- LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
-
- // Verify an invalid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- tokenId = new OzoneBlockTokenIdentifier("", "",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- LOG.info("Unsigned token {} is {}", tokenId,
- verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
-
- }
-
- public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- PrivateKey privateKey) throws NoSuchAlgorithmException,
- InvalidKeyException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initSign(privateKey);
- rsaSignature.update(tokenId.getBytes());
- byte[] signature = rsaSignature.sign();
- return signature;
- }
-
- public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- byte[] signature, Certificate certificate) throws InvalidKeyException,
- NoSuchAlgorithmException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initVerify(certificate);
- rsaSignature.update(tokenId.getBytes());
- boolean isValid = rsaSignature.verify(signature);
- return isValid;
- }
-
- private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
- Mac mac, SecretKey key) {
- try {
- mac.init(key);
- } catch (InvalidKeyException ike) {
- throw new IllegalArgumentException("Invalid key to HMAC computation",
- ike);
- }
- return mac.doFinal(identifier.getBytes());
- }
-
- OzoneBlockTokenIdentifier generateTestToken() {
- return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
- RandomStringUtils.randomAlphabetic(5),
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- }
-
- @Test
- public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
- CertificateEncodingException, NoSuchProviderException,
- InvalidKeyException, SignatureException {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordAsym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
-
- // Create Ozone Master certificate (SCM CA issued cert) and key store
- X509Certificate certificate;
- certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
- kp, 30, "SHA256withRSA");
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordAsym.add(
- signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
- duration / testTokenCount);
-
- startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
- certificate);
- }
- duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
- + "is {} ns", duration / testTokenCount);
- }
-
- @Test
- public void testSymmetricTokenPerf() {
- String hmacSHA1 = "HmacSHA1";
- String hmacSHA256 = "HmacSHA256";
-
- testSymmetricTokenPerfHelper(hmacSHA1, 64);
- testSymmetricTokenPerfHelper(hmacSHA256, 1024);
- }
-
- public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordSym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyGenerator keyGen;
- try {
- keyGen = KeyGenerator.getInstance(hmacAlgorithm);
- keyGen.init(keyLen);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- Mac mac;
- try {
- mac = Mac.getInstance(hmacAlgorithm);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- SecretKey secretKey = keyGen.generateKey();
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordSym.add(
- signTokenSymmetric(tokenIds.get(i), mac, secretKey));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
- hmacAlgorithm, keyLen, duration / testTokenCount);
- }
-
- // TODO: verify certificate with a trust store
- public boolean verifyCert(Certificate certificate) {
- return true;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-9069. Fix
SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S
T.
Posted by xy...@apache.org.
YARN-9069. Fix SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07142f54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07142f54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07142f54
Branch: refs/heads/HDDS-4
Commit: 07142f54a8c7f70857e99c041f3a2a5189c809b5
Parents: a68d766
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 22:02:59 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 22:02:59 2018 +0530
----------------------------------------------------------------------
.../yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07142f54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 163f707..ede0d15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -54,6 +54,8 @@ public class SchedulerInfo {
this.schedulerName = "Fair Scheduler";
} else if (rs instanceof FifoScheduler) {
this.schedulerName = "Fifo Scheduler";
+ } else {
+ this.schedulerName = rs.getClass().getSimpleName();
}
this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-14927. ITestS3GuardTool
failures in testDestroyNoBucket(). Contributed by Gabor Bota.
Posted by xy...@apache.org.
HADOOP-14927. ITestS3GuardTool failures in testDestroyNoBucket(). Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb0d3a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb0d3a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb0d3a3
Branch: refs/heads/HDDS-4
Commit: 7eb0d3a32435da110dc9e6004dba8c5c9b082c35
Parents: 184cced
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Nov 28 16:57:12 2018 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Nov 29 09:36:39 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 38 ++++++++++++--------
1 file changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb0d3a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 1316121..aea57a6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -218,6 +218,27 @@ public abstract class S3GuardTool extends Configured implements Tool {
format.addOptionWithValue(SECONDS_FLAG);
}
+ protected void checkMetadataStoreUri(List<String> paths) throws IOException {
+ // be sure that path is provided in params, so there's no IOoBE
+ String s3Path = "";
+ if(!paths.isEmpty()) {
+ s3Path = paths.get(0);
+ }
+
+ // Check if DynamoDB url is set from arguments.
+ String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
+ if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
+ // If not set, check if filesystem is guarded by creating an
+ // S3AFileSystem and check if hasMetadataStore is true
+ try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
+ S3AFileSystem.newInstance(toUri(s3Path), getConf())){
+ Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
+ "The S3 bucket is unguarded. " + getName()
+ + " can not be used on an unguarded bucket.");
+ }
+ }
+ }
+
/**
* Parse metadata store from command line option or HDFS configuration.
*
@@ -500,20 +521,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
- String s3Path = paths.get(0);
-
- // Check if DynamoDB url is set from arguments.
- String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
- if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
- // If not set, check if filesystem is guarded by creating an
- // S3AFileSystem and check if hasMetadataStore is true
- try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
- S3AFileSystem.newInstance(toUri(s3Path), getConf())){
- Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
- "The S3 bucket is unguarded. " + getName()
- + " can not be used on an unguarded bucket.");
- }
- }
+ checkMetadataStoreUri(paths);
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {
@@ -590,6 +598,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
throw e;
}
+ checkMetadataStoreUri(paths);
+
try {
initMetadataStore(false);
} catch (FileNotFoundException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HDDS-696. Bootstrap genesis
SCM(CA) with self-signed certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
HDDS-696. Bootstrap genesis SCM(CA) with self-signed certificate.
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f51d23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f51d23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f51d23
Branch: refs/heads/HDDS-4
Commit: 87f51d23d9899e46ac31058b726b272f1a7880ba
Parents: 9b9b8e4
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Nov 27 15:02:07 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:58:54 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 57 ++-
.../hdds/security/x509/SecurityConfig.java | 84 ++++-
.../authority/CertificateServer.java | 11 +-
.../certificate/authority/DefaultCAServer.java | 373 +++++++++++++++++++
.../certificate/client/CertificateClient.java | 18 +-
.../certificate/utils/CertificateCodec.java | 280 ++++++++++++++
.../x509/certificate/utils/package-info.java | 22 ++
.../certificates/CertificateSignRequest.java | 245 ------------
.../certificates/SelfSignedCertificate.java | 212 -----------
.../x509/certificates/package-info.java | 22 --
.../utils/CertificateSignRequest.java | 245 ++++++++++++
.../utils/SelfSignedCertificate.java | 238 ++++++++++++
.../x509/certificates/utils/package-info.java | 22 ++
.../security/x509/keys/HDDSKeyGenerator.java | 32 +-
.../security/x509/keys/HDDSKeyPEMWriter.java | 255 -------------
.../hdds/security/x509/keys/KeyCodec.java | 337 +++++++++++++++++
.../apache/hadoop/ozone/common/StorageInfo.java | 2 +-
.../authority/TestDefaultCAServer.java | 118 ++++++
.../certificate/authority/package-info.java | 22 ++
.../certificate/utils/TestCertificateCodec.java | 218 +++++++++++
.../TestCertificateSignRequest.java | 19 +-
.../x509/certificates/TestRootCertificate.java | 46 +--
.../x509/certificates/package-info.java | 4 +-
.../x509/keys/TestHDDSKeyPEMWriter.java | 216 -----------
.../hdds/security/x509/keys/TestKeyCodec.java | 216 +++++++++++
.../hadoop/utils/db/TestDBStoreBuilder.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 15 +-
27 files changed, 2267 insertions(+), 1067 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 553f4aa..a02152d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -1,19 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
*/
package org.apache.hadoop.hdds;
@@ -25,61 +24,45 @@ import org.apache.hadoop.utils.db.DBProfile;
*/
public final class HddsConfigKeys {
- /**
- * Do not instantiate.
- */
- private HddsConfigKeys() {
- }
-
public static final String HDDS_HEARTBEAT_INTERVAL =
"hdds.heartbeat.interval";
public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT =
"30s";
-
public static final String HDDS_NODE_REPORT_INTERVAL =
"hdds.node.report.interval";
public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
"60s";
-
public static final String HDDS_CONTAINER_REPORT_INTERVAL =
"hdds.container.report.interval";
public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
"60s";
-
public static final String HDDS_PIPELINE_REPORT_INTERVAL =
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
"60s";
-
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
"60s";
-
public static final String HDDS_CONTAINER_ACTION_MAX_LIMIT =
"hdds.container.action.max.limit";
public static final int HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT =
20;
-
public static final String HDDS_PIPELINE_ACTION_MAX_LIMIT =
"hdds.pipeline.action.max.limit";
public static final int HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT =
20;
-
// Configuration to allow volume choosing policy.
public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
"hdds.datanode.volume.choosing.policy";
-
// DB Profiles used by ROCKDB instances.
public static final String HDDS_DB_PROFILE = "hdds.db.profile";
public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
-
// Once a container usage crosses this threshold, it is eligible for
// closing.
public static final String HDDS_CONTAINER_CLOSE_THRESHOLD =
"hdds.container.close.threshold";
public static final float HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
-
public static final String HDDS_SCM_CHILLMODE_ENABLED =
"hdds.scm.chillmode.enabled";
public static final boolean HDDS_SCM_CHILLMODE_ENABLED_DEFAULT = true;
@@ -97,11 +80,9 @@ public final class HddsConfigKeys {
public static final String HDDS_SCM_CHILLMODE_THRESHOLD_PCT =
"hdds.scm.chillmode.threshold.pct";
public static final double HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT = 0.99;
-
public static final String HDDS_LOCK_MAX_CONCURRENCY =
"hdds.lock.max.concurrency";
public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
-
// This configuration setting is used as a fallback location by all
// Ozone/HDDS services for their metadata. It is useful as a single
// config point for test/PoC clusters.
@@ -121,7 +102,6 @@ public final class HddsConfigKeys {
public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
-
// TODO : Talk to StorageIO classes and see if they can return a secure
// storage location for each node.
public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
@@ -131,7 +111,6 @@ public final class HddsConfigKeys {
public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ ".name";
public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
-
/**
* Maximum duration of certificates issued by SCM including Self-Signed Roots.
* The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
@@ -140,12 +119,20 @@ public final class HddsConfigKeys {
public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
// Limit Certificate duration to a max value of 5 years.
public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
-
public static final String HDDS_X509_SIGNATURE_ALGO =
"hdds.x509.signature.algorithm";
public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
-
- public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED = "hdds.grpc.block" +
- ".token.enabled";
+ public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED =
+ "hdds.grpc.block.token.enabled";
public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
+ public static final String HDDS_X509_DIR_NAME = "hdds.x509.dir.name";
+ public static final String HDDS_X509_DIR_NAME_DEFAULT = "certs";
+ public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
+ public static final String HDDS_X509_FILE_NAME_DEFAULT = "certificate.crt";
+
+ /**
+ * Do not instantiate.
+ */
+ private HddsConfigKeys() {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 2826e55..ee20a21 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -46,6 +46,10 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_FILE_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
@@ -54,7 +58,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* A class that deals with all Security related configs in HDDS.
- *
+ * <p>
* This class allows security configs to be read and used consistently across
* all of security related code base.
*/
@@ -73,6 +77,8 @@ public class SecurityConfig {
private final Duration certDuration;
private final String x509SignatureAlgo;
private final Boolean grpcBlockTokenEnabled;
+ private final String certificateDir;
+ private final String certificateFileName;
/**
* Constructs a SecurityConfig.
@@ -108,6 +114,10 @@ public class SecurityConfig {
this.certDuration = Duration.parse(durationString);
this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+ this.certificateDir = this.configuration.get(HDDS_X509_DIR_NAME,
+ HDDS_X509_DIR_NAME_DEFAULT);
+ this.certificateFileName = this.configuration.get(HDDS_X509_FILE_NAME,
+ HDDS_X509_FILE_NAME_DEFAULT);
this.grpcBlockTokenEnabled = this.configuration.getBoolean(
HDDS_GRPC_BLOCK_TOKEN_ENABLED,
@@ -127,8 +137,17 @@ public class SecurityConfig {
}
/**
- * Returns the public key file name, This is used for storing the public
- * keys on disk.
+ * Returns the Standard Certificate file name.
+ *
+ * @return String - Name of the Certificate File.
+ */
+ public String getCertificateFileName() {
+ return certificateFileName;
+ }
+
+ /**
+ * Returns the public key file name, This is used for storing the public keys
+ * on disk.
*
* @return String, File name used for public keys.
*/
@@ -137,8 +156,8 @@ public class SecurityConfig {
}
/**
- * Returns the private key file name.This is used for storing the private
- * keys on disk.
+ * Returns the private key file name.This is used for storing the private keys
+ * on disk.
*
* @return String, File name used for private keys.
*/
@@ -149,16 +168,47 @@ public class SecurityConfig {
/**
* Returns the File path to where keys are stored.
*
- * @return String Key location.
+ * @return path Key location.
*/
public Path getKeyLocation() {
return Paths.get(metadatDir, keyDir);
}
/**
+ * Returns the File path to where keys are stored with an additional component
+ * name inserted in between.
+ *
+ * @param component - Component Name - String.
+ * @return Path location.
+ */
+ public Path getKeyLocation(String component) {
+ return Paths.get(metadatDir, component, keyDir);
+ }
+
+ /**
+ * Returns the File path to where keys are stored.
+ *
+ * @return path Key location.
+ */
+ public Path getCertificateLocation() {
+ return Paths.get(metadatDir, certificateDir);
+ }
+
+ /**
+ * Returns the File path to where keys are stored with an addition component
+ * name inserted in between.
+ *
+ * @param component - Component Name - String.
+ * @return Path location.
+ */
+ public Path getCertificateLocation(String component) {
+ return Paths.get(metadatDir, component, certificateDir);
+ }
+
+ /**
* Gets the Key Size, The default key size is 2048, since the default
- * algorithm used is RSA. User can change this by setting the "hdds.key
- * .len" in configuration.
+ * algorithm used is RSA. User can change this by setting the "hdds.key.len"
+ * in configuration.
*
* @return key size.
*/
@@ -177,8 +227,8 @@ public class SecurityConfig {
}
/**
- * Returns the Key generation Algorithm used. User can change this by
- * setting the "hdds.key.algo" in configuration.
+ * Returns the Key generation Algorithm used. User can change this by setting
+ * the "hdds.key.algo" in configuration.
*
* @return String Algo.
*/
@@ -188,8 +238,8 @@ public class SecurityConfig {
/**
* Returns the X.509 Signature Algorithm used. This can be changed by setting
- * "hdds.x509.signature.algorithm" to the new name. The default algorithm
- * is SHA256withRSA.
+ * "hdds.x509.signature.algorithm" to the new name. The default algorithm is
+ * SHA256withRSA.
*
* @return String
*/
@@ -207,11 +257,11 @@ public class SecurityConfig {
}
/**
- * Returns the maximum length a certificate can be valid in SCM. The
- * default value is 5 years. This can be changed by setting
- * "hdds.x509.max.duration" in configuration. The formats accepted are
- * based on the ISO-8601 duration format PnDTnHnMn.nS
- *
+ * Returns the maximum length a certificate can be valid in SCM. The default
+ * value is 5 years. This can be changed by setting "hdds.x509.max.duration"
+ * in configuration. The formats accepted are based on the ISO-8601 duration
+ * format PnDTnHnMn.nS
+ * <p>
* Default value is 5 years and written as P1865D.
*
* @return Duration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index 5b5deef..ee685c8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -20,10 +20,12 @@
package org.apache.hadoop.hdds.security.x509.certificate.authority;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.bouncycastle.cert.X509CertificateHolder;
+import java.io.IOException;
+import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.Future;
@@ -47,16 +49,19 @@ public interface CertificateServer {
* Returns the CA Certificate for this CA.
*
* @return X509CertificateHolder - Certificate for this CA.
- * @throws SCMSecurityException -- usually thrown if this CA is not
+ * @throws CertificateException - usually thrown if this CA is not
* initialized.
+ * @throws IOException - on Error.
*/
X509CertificateHolder getCACertificate()
- throws SCMSecurityException;
+ throws CertificateException, IOException;
/**
* Request a Certificate based on Certificate Signing Request.
*
* @param csr - Certificate Signing Request.
+ * @param approver - An Enum which says what kind of approval process to
+ * follow.
* @return A future that will have this certificate when this request is
* approved.
* @throws SCMSecurityException - on Error.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
new file mode 100644
index 0000000..f6e6e5c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.concurrent.Future;
+import java.util.function.Consumer;
+
+/**
+ * The default CertificateServer used by SCM. This has no dependencies on any
+ * external system, this allows us to bootstrap a CertificateServer from
+ * Scratch.
+ * <p>
+ * Details =======
+ * <p>
+ * The Default CA server is one of the many possible implementations of an SCM
+ * Certificate Authority.
+ * <p>
+ * A certificate authority needs the Root Certificates and its private key to
+ * operate. The init function of the DefaultCA Server detects four possible
+ * states the System can be in.
+ * <p>
+ * 1. Success - This means that the expected Certificates and Keys are in
+ * place, and the CA was able to read those files into memory.
+ * <p>
+ * 2. Missing Keys - This means that private keys are missing. This is an error
+ * state which SCM CA cannot recover from. The cluster might have been
+ * initialized earlier and for some reason, we are not able to find the private
+ * keys for the CA. Eventually we will have 2 ways to recover from this state,
+ * first one is to copy the SCM CA private keys from a backup. Second one is to
+ * rekey the whole cluster. Both of these are improvements we will support in
+ * future.
+ * <p>
+ * 3. Missing Certificate - Similar to Missing Keys, but the root certificates
+ * are missing.
+ * <p>
+ * 4. Initialize - We don't have keys or certificates. DefaultCA assumes that
+ * this is a system bootup and will generate the keys and certificates
+ * automatically.
+ * <p>
+ * The init() follows the following logic,
+ * <p>
+ * 1. Compute the Verification Status -- Success, Missing Keys, Missing Certs or
+ * Initialize.
+ * <p>
+ * 2. ProcessVerificationStatus - Returns a Lambda, based on the Verification
+ * Status.
+ * <p>
+ * 3. Invoke the Lambda function.
+ * <p>
+ * At the end of the init function, we have functional CA. This function can be
+ * invoked as many times since we will regenerate the keys and certs only if
+ * both of them are missing.
+ */
+public class DefaultCAServer implements CertificateServer {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultCAServer.class);
+ private final String subject;
+ private final String clusterID;
+ private final String scmID;
+ private String componentName = Paths.get("scm", "ca").toString();
+ private Path caKeysPath;
+ private Path caRootX509Path;
+ private SecurityConfig config;
+
+ /**
+ * Create an Instance of DefaultCAServer.
+ *
+ * @param subject - String Subject
+ * @param clusterID - String ClusterID
+ * @param scmID - String SCMID.
+ */
+ public DefaultCAServer(String subject, String clusterID, String scmID) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ }
+
+ @Override
+ public void init(SecurityConfig securityConfig, CAType type)
+ throws SCMSecurityException {
+ caKeysPath = securityConfig.getKeyLocation(componentName);
+ caRootX509Path = securityConfig.getCertificateLocation(componentName);
+ this.config = securityConfig;
+
+ /* In future we will spilt this code to have different kind of CAs.
+ * Right now, we have only self-signed CertificateServer.
+ */
+
+ if (type == CAType.SELF_SIGNED_CA) {
+ VerificationStatus status = verifySelfSignedCA(securityConfig);
+ Consumer<SecurityConfig> caInitializer =
+ processVerificationStatus(status);
+ caInitializer.accept(securityConfig);
+ return;
+ }
+
+ LOG.error("We support only Self-Signed CAs for now.");
+ throw new IllegalStateException("Not implemented functionality requested.");
+ }
+
+ @Override
+ public X509CertificateHolder getCACertificate() throws
+ CertificateException, IOException {
+ CertificateCodec certificateCodec =
+ new CertificateCodec(config, componentName);
+ return certificateCodec.readCertificate();
+ }
+
+ @Override
+ public Future<X509CertificateHolder> requestCertificate(
+ CertificateSignRequest csr, CertificateApprover approver)
+ throws SCMSecurityException {
+ return null;
+ }
+
+ @Override
+ public Future<Boolean> revokeCertificate(X509Certificate certificate,
+ CertificateApprover approver) throws SCMSecurityException {
+ return null;
+ }
+
+ /**
+ * Generates a Self Signed CertificateServer. These are the steps in
+ * generating a Self-Signed CertificateServer.
+ * <p>
+ * 1. Generate a Private/Public Key Pair. 2. Persist to a protected location.
+ * 3. Generate a SelfSigned Root CertificateServer certificate.
+ *
+ * @param securityConfig - Config.
+ */
+ private void generateSelfSignedCA(SecurityConfig securityConfig) throws
+ NoSuchAlgorithmException, NoSuchProviderException, IOException {
+ KeyPair keyPair = generateKeys(securityConfig);
+ generateRootCertificate(securityConfig, keyPair);
+ }
+
+ /**
+ * Verify Self-Signed CertificateServer. 1. Check if the Certificate exist. 2.
+ * Check if the key pair exists.
+ *
+ * @param securityConfig -- Config
+ * @return Verification Status
+ */
+ private VerificationStatus verifySelfSignedCA(SecurityConfig securityConfig) {
+ /*
+ The following is the truth table for the States.
+ True means we have that file False means it is missing.
+ +--------------+--------+--------+--------------+
+ | Certificates | Keys | Result | Function |
+ +--------------+--------+--------+--------------+
+ | True | True | True | Success |
+ | False | False | True | Initialize |
+ | True | False | False | Missing Key |
+ | False | True | False | Missing Cert |
+ +--------------+--------+--------+--------------+
+
+ This truth table maps to ~(certs xor keys) or certs == keys
+ */
+ boolean keyStatus = checkIfKeysExist();
+ boolean certStatus = checkIfCertificatesExist();
+
+ if ((certStatus == keyStatus) && (certStatus)) {
+ return VerificationStatus.SUCCESS;
+ }
+
+ if ((certStatus == keyStatus) && (!certStatus)) {
+ return VerificationStatus.INITIALIZE;
+ }
+
+ // At this point certStatus is not equal to keyStatus.
+ if (certStatus) {
+ return VerificationStatus.MISSING_KEYS;
+ }
+
+ return VerificationStatus.MISSING_CERTIFICATE;
+ }
+
+ /**
+ * Returns Keys status.
+ *
+ * @return True if the key files exist.
+ */
+ private boolean checkIfKeysExist() {
+ if (!Files.exists(caKeysPath)) {
+ return false;
+ }
+
+ if (!Files.exists(Paths.get(caKeysPath.toString(),
+ this.config.getPrivateKeyFileName()))) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns certificate Status.
+ *
+ * @return True if the Certificate files exist.
+ */
+ private boolean checkIfCertificatesExist() {
+ if (!Files.exists(caRootX509Path)) {
+ return false;
+ }
+ if (!Files.exists(Paths.get(caRootX509Path.toString(),
+ this.config.getCertificateFileName()))) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Based on the Status of the verification, we return a lambda that gets
+ * executed by the init function of the CA.
+ *
+ * @param status - Verification Status.
+ */
+ @VisibleForTesting
+ Consumer<SecurityConfig> processVerificationStatus(
+ VerificationStatus status) {
+ Consumer<SecurityConfig> consumer = null;
+ switch (status) {
+ case SUCCESS:
+ consumer = (arg) -> LOG.info("CertificateServer validation is " +
+ "successful");
+ break;
+ case MISSING_KEYS:
+ consumer = (arg) -> {
+ LOG.error("We have found the Certificate for this CertificateServer, " +
+ "but keys used by this CertificateServer is missing. This is a " +
+ "non-recoverable error. Please restart the system after locating " +
+ "the Keys used by the CertificateServer.");
+ LOG.error("Exiting due to unrecoverable CertificateServer error.");
+ throw new IllegalStateException("Missing Keys, cannot continue.");
+ };
+ break;
+ case MISSING_CERTIFICATE:
+ consumer = (arg) -> {
+ LOG.error("We found the keys, but the root certificate for this " +
+ "CertificateServer is missing. Please restart SCM after locating " +
+ "the " +
+ "Certificates.");
+ LOG.error("Exiting due to unrecoverable CertificateServer error.");
+ throw new IllegalStateException("Missing Root Certs, cannot continue.");
+ };
+ break;
+ case INITIALIZE:
+ consumer = (arg) -> {
+ try {
+ generateSelfSignedCA(arg);
+ } catch (NoSuchProviderException | NoSuchAlgorithmException
+ | IOException e) {
+ LOG.error("Unable to initialize CertificateServer.", e);
+ }
+ VerificationStatus newStatus = verifySelfSignedCA(arg);
+ if (newStatus != VerificationStatus.SUCCESS) {
+ LOG.error("Unable to initialize CertificateServer, failed in " +
+ "verification.");
+ }
+ };
+ break;
+ default:
+ /* Make CheckStyle happy */
+ break;
+ }
+ return consumer;
+ }
+
+ /**
+ * Generates a KeyPair for the Certificate.
+ *
+ * @param securityConfig - SecurityConfig.
+ * @return Key Pair.
+ * @throws NoSuchProviderException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws IOException - on Error.
+ */
+ private KeyPair generateKeys(SecurityConfig securityConfig)
+ throws NoSuchProviderException, NoSuchAlgorithmException, IOException {
+ HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig);
+ KeyPair keys = keyGenerator.generateKey();
+ KeyCodec keyPEMWriter = new KeyCodec(securityConfig,
+ componentName);
+ keyPEMWriter.writeKey(keys);
+ return keys;
+ }
+
+ /**
+ * Generates a self-signed Root Certificate for CA.
+ *
+ * @param securityConfig - SecurityConfig
+ * @param key - KeyPair.
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ */
+ private void generateRootCertificate(SecurityConfig securityConfig,
+ KeyPair key) throws IOException, SCMSecurityException {
+ Preconditions.checkNotNull(this.config);
+ LocalDate beginDate = LocalDate.now().atStartOfDay().toLocalDate();
+ LocalDateTime temp = LocalDateTime.of(beginDate, LocalTime.MIDNIGHT);
+ LocalDate endDate =
+ temp.plus(securityConfig.getMaxCertificateDuration()).toLocalDate();
+ X509CertificateHolder selfSignedCertificate =
+ SelfSignedCertificate
+ .newBuilder()
+ .setSubject(this.subject)
+ .setScmID(this.scmID)
+ .setClusterID(this.clusterID)
+ .setBeginDate(beginDate)
+ .setEndDate(endDate)
+ .makeCA()
+ .setConfiguration(securityConfig.getConfiguration())
+ .setKey(key)
+ .build();
+
+ CertificateCodec certCodec =
+ new CertificateCodec(config, componentName);
+ certCodec.writeCertificate(selfSignedCertificate);
+ }
+
+ /**
+ * This represents the verification status of the CA. Based on this enum
+ * appropriate action is taken in the Init.
+ */
+ @VisibleForTesting
+ enum VerificationStatus {
+ SUCCESS, /* All artifacts needed by CertificateServer is present */
+ MISSING_KEYS, /* Private key is missing, certificate Exists.*/
+ MISSING_CERTIFICATE, /* Keys exist, but root certificate missing.*/
+ INITIALIZE /* All artifacts are missing, we should init the system. */
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index bfc0576..e33c9b6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.security.x509.certificate.client;
-import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import java.io.InputStream;
@@ -64,7 +64,7 @@ public interface CertificateClient {
/**
* Verifies if this certificate is part of a trusted chain.
- *
+ * @param certificate - certificate.
* @return true if it trusted, false otherwise.
*/
boolean verifyCertificate(X509Certificate certificate);
@@ -74,7 +74,9 @@ public interface CertificateClient {
* key.
*
* @param stream - Data stream to sign.
+ * @param component - name of the component.
* @return byte array - containing the signature.
+ * @throws CertificateException - on Error.
*/
byte[] signDataStream(InputStream stream, String component)
throws CertificateException;
@@ -82,6 +84,7 @@ public interface CertificateClient {
/**
* Verifies a digital Signature, given the signature and the certificate of
* the signer.
+ *
* @param stream - Data Stream.
* @param signature - Byte Array containing the signature.
* @param cert - Certificate of the Signer.
@@ -123,7 +126,7 @@ public interface CertificateClient {
*
* @param key - private key
* @param component - name of the component.
- * @throws CertificateException
+ * @throws CertificateException - on Error.
*/
void storePrivateKey(PrivateKey key, String component)
throws CertificateException;
@@ -132,7 +135,8 @@ public interface CertificateClient {
* Stores the public key of a specified component.
*
* @param key - public key
- * @throws CertificateException
+ * @param component - name of the component.
+ * @throws CertificateException - on Error.
*/
void storePublicKey(PublicKey key, String component)
throws CertificateException;
@@ -142,7 +146,7 @@ public interface CertificateClient {
*
* @param certificate - X509 Certificate
* @param component - Name of the component.
- * @throws CertificateException
+ * @throws CertificateException - on Error.
*/
void storeCertificate(X509Certificate certificate, String component)
throws CertificateException;
@@ -152,7 +156,7 @@ public interface CertificateClient {
*
* @param certStore - Cert Store.
* @param component - Trust Chain.
- * @throws CertificateException
+ * @throws CertificateException - on Error.
*/
void storeTrustChain(CertStore certStore,
String component) throws CertificateException;
@@ -162,7 +166,7 @@ public interface CertificateClient {
*
* @param certificates - List of Certificates.
* @param component - String component.
- * @throws CertificateException
+ * @throws CertificateException - on Error.
*/
void storeTrustChain(List<X509Certificate> certificates,
String component) throws CertificateException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
new file mode 100644
index 0000000..b2a37b5
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * A class used to read and write X.509 certificates PEM encoded Streams.
+ */
+public class CertificateCodec {
+ public static final String BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
+ public static final String END_CERT = "-----END CERTIFICATE-----";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CertificateCodec.class);
+ private static final JcaX509CertificateConverter CERTIFICATE_CONVERTER
+ = new JcaX509CertificateConverter();
+ private final SecurityConfig securityConfig;
+ private final Path location;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ /**
+ * Creates an CertificateCodec.
+ *
+ * @param config - Security Config.
+ * @param component - Component String.
+ */
+ public CertificateCodec(SecurityConfig config, String component) {
+ this.securityConfig = config;
+ this.location = securityConfig.getCertificateLocation(component);
+ }
+
+ /**
+ * Creates an CertificateCodec.
+ *
+ * @param configuration - Configuration
+ */
+ public CertificateCodec(Configuration configuration) {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ this.location = securityConfig.getCertificateLocation();
+ }
+
+ /**
+ * Returns a X509 Certificate from the Certificate Holder.
+ *
+ * @param holder - Holder
+ * @return X509Certificate.
+ * @throws CertificateException - on Error.
+ */
+ public static X509Certificate getX509Certificate(X509CertificateHolder holder)
+ throws CertificateException {
+ return CERTIFICATE_CONVERTER.getCertificate(holder);
+ }
+
+ /**
+ * Get Certificate location.
+ *
+ * @return Path
+ */
+ public Path getLocation() {
+ return location;
+ }
+
+ /**
+ * Returns the Certificate as a PEM encoded String.
+ *
+ * @param x509CertHolder - X.509 Certificate Holder.
+ * @return PEM Encoded Certificate String.
+ * @throws SCMSecurityException - On failure to create a PEM String.
+ */
+ public String getPEMEncodedString(X509CertificateHolder x509CertHolder)
+ throws SCMSecurityException {
+ try {
+ StringWriter stringWriter = new StringWriter();
+ try (JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
+ pemWriter.writeObject(getX509Certificate(x509CertHolder));
+ }
+ return stringWriter.toString();
+ } catch (CertificateException | IOException e) {
+ LOG.error("Error in encoding certificate." + x509CertHolder
+ .getSubject().toString(), e);
+ throw new SCMSecurityException("PEM Encoding failed for certificate." +
+ x509CertHolder.getSubject().toString(), e);
+ }
+ }
+
+ /**
+ * Gets the X.509 Certificate from PEM encoded String.
+ *
+ * @param pemEncodedString - PEM encoded String.
+ * @return X509Certificate - Certificate.
+ * @throws CertificateException - Thrown on Failure.
+ * @throws IOException - Thrown on Failure.
+ */
+ public X509Certificate getX509Certificate(String pemEncodedString)
+ throws CertificateException, IOException {
+ CertificateFactory fact = CertificateFactory.getInstance("X.509");
+ try (InputStream input = IOUtils.toInputStream(pemEncodedString, UTF_8)) {
+ return (X509Certificate) fact.generateCertificate(input);
+ }
+ }
+
+ /**
+ * Write the Certificate pointed to the location by the configs.
+ *
+ * @param xCertificate - Certificate to write.
+ * @throws SCMSecurityException - on Error.
+ * @throws IOException - on Error.
+ */
+ public void writeCertificate(X509CertificateHolder xCertificate)
+ throws SCMSecurityException, IOException {
+ String pem = getPEMEncodedString(xCertificate);
+ writeCertificate(location.toAbsolutePath(),
+ this.securityConfig.getCertificateFileName(), pem, false);
+ }
+
+ /**
+ * Write the Certificate to the specific file.
+ *
+ * @param xCertificate - Certificate to write.
+ * @param fileName - file name to write to.
+ * @param overwrite - boolean value, true means overwrite an existing
+ * certificate.
+ * @throws SCMSecurityException - On Error.
+ * @throws IOException - On Error.
+ */
+ public void writeCertificate(X509CertificateHolder xCertificate,
+ String fileName, boolean overwrite)
+ throws SCMSecurityException, IOException {
+ String pem = getPEMEncodedString(xCertificate);
+ writeCertificate(location.toAbsolutePath(), fileName, pem, overwrite);
+ }
+
+ /**
+ * Helper function that writes data to the file.
+ *
+ * @param basePath - Base Path where the file needs to written to.
+ * @param fileName - Certificate file name.
+ * @param pemEncodedCertificate - pemEncoded Certificate file.
+ * @param force - Overwrite if the file exists.
+ * @throws IOException - on Error.
+ */
+ public synchronized void writeCertificate(Path basePath, String fileName,
+ String pemEncodedCertificate, boolean force)
+ throws IOException {
+ File certificateFile =
+ Paths.get(basePath.toString(), fileName).toFile();
+ if (certificateFile.exists() && !force) {
+ throw new SCMSecurityException("Specified certificate file already " +
+ "exists.Please use force option if you want to overwrite it.");
+ }
+ if (!basePath.toFile().exists()) {
+ if (!basePath.toFile().mkdirs()) {
+ LOG.error("Unable to create file path. Path: {}", basePath);
+ throw new IOException("Creation of the directories failed."
+ + basePath.toString());
+ }
+ }
+ try (FileOutputStream file = new FileOutputStream(certificateFile)) {
+ IOUtils.write(pemEncodedCertificate, file, UTF_8);
+ }
+
+ Files.setPosixFilePermissions(certificateFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Rertuns a default certificate using the default paths for this component.
+ *
+ * @return X509CertificateHolder.
+ * @throws SCMSecurityException - on Error.
+ * @throws CertificateException - on Error.
+ * @throws IOException - on Error.
+ */
+ public X509CertificateHolder readCertificate() throws
+ CertificateException, IOException {
+ return readCertificate(this.location.toAbsolutePath(),
+ this.securityConfig.getCertificateFileName());
+ }
+
+ /**
+ * Returns the certificate from the specific PEM encoded file.
+ *
+ * @param basePath - base path
+ * @param fileName - fileName
+ * @return X%09 Certificate
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ * @throws CertificateException - on Error.
+ */
+ public synchronized X509CertificateHolder readCertificate(Path basePath,
+ String fileName) throws IOException, CertificateException {
+ File certificateFile = Paths.get(basePath.toString(), fileName).toFile();
+ return getX509CertificateHolder(certificateFile);
+ }
+
+ /**
+ * Helper function to read certificate.
+ *
+ * @param certificateFile - Full path to certificate file.
+ * @return X509CertificateHolder
+ * @throws IOException - On Error.
+ * @throws CertificateException - On Error.
+ */
+ private X509CertificateHolder getX509CertificateHolder(File certificateFile)
+ throws IOException, CertificateException {
+ if (!certificateFile.exists()) {
+ throw new IOException("Unable to find the requested certificate. Path: "
+ + certificateFile.toString());
+ }
+ CertificateFactory fact = CertificateFactory.getInstance("X.509");
+ try (FileInputStream is = new FileInputStream(certificateFile)) {
+ return getCertificateHolder(
+ (X509Certificate) fact.generateCertificate(is));
+ }
+ }
+
+ /**
+ * Returns the Certificate holder from X509Ceritificate class.
+ *
+ * @param x509cert - Certificate class.
+ * @return X509CertificateHolder
+ * @throws CertificateEncodingException - on Error.
+ * @throws IOException - on Error.
+ */
+ public X509CertificateHolder getCertificateHolder(X509Certificate x509cert)
+ throws CertificateEncodingException, IOException {
+ return new X509CertificateHolder(x509cert.getEncoded());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
new file mode 100644
index 0000000..4971d4a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Certificate Utils.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
deleted file mode 100644
index 0e762a5..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509.certificates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
-import org.apache.logging.log4j.util.Strings;
-import org.bouncycastle.asn1.DEROctetString;
-import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.BasicConstraints;
-import org.bouncycastle.asn1.x509.Extension;
-import org.bouncycastle.asn1.x509.Extensions;
-import org.bouncycastle.asn1.x509.GeneralName;
-import org.bouncycastle.asn1.x509.GeneralNames;
-import org.bouncycastle.asn1.x509.KeyUsage;
-import org.bouncycastle.operator.ContentSigner;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-import org.bouncycastle.pkcs.PKCS10CertificationRequest;
-import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
-import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
-
-import java.io.IOException;
-import java.security.KeyPair;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-/**
- * A certificate sign request object that wraps operations to build a
- * PKCS10CertificationRequest to CA.
- */
-public final class CertificateSignRequest {
- private final KeyPair keyPair;
- private final SecurityConfig config;
- private final Extensions extensions;
- private String subject;
- private String clusterID;
- private String scmID;
-
- /**
- * Private Ctor for CSR.
- *
- * @param subject - Subject
- * @param scmID - SCM ID
- * @param clusterID - Cluster ID
- * @param keyPair - KeyPair
- * @param config - SCM Config
- * @param extensions - CSR extensions
- */
- private CertificateSignRequest(String subject, String scmID, String clusterID,
- KeyPair keyPair, SecurityConfig config, Extensions extensions) {
- this.subject = subject;
- this.clusterID = clusterID;
- this.scmID = scmID;
- this.keyPair = keyPair;
- this.config = config;
- this.extensions = extensions;
- }
-
- private PKCS10CertificationRequest generateCSR() throws
- OperatorCreationException {
- X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
- clusterID);
- PKCS10CertificationRequestBuilder p10Builder =
- new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
-
- ContentSigner contentSigner =
- new JcaContentSignerBuilder(config.getSignatureAlgo())
- .setProvider(config.getProvider())
- .build(keyPair.getPrivate());
-
- if (extensions != null) {
- p10Builder.addAttribute(
- PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
- }
- return p10Builder.build(contentSigner);
- }
-
- /**
- * Builder class for Certificate Sign Request.
- */
- public static class Builder {
- private String subject;
- private String clusterID;
- private String scmID;
- private KeyPair key;
- private SecurityConfig config;
- private List<GeneralName> altNames;
- private Boolean ca = false;
-
- public CertificateSignRequest.Builder setConfiguration(
- Configuration configuration) {
- this.config = new SecurityConfig(configuration);
- return this;
- }
-
- public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
- this.key = keyPair;
- return this;
- }
-
- public CertificateSignRequest.Builder setSubject(String subjectString) {
- this.subject = subjectString;
- return this;
- }
-
- public CertificateSignRequest.Builder setClusterID(String s) {
- this.clusterID = s;
- return this;
- }
-
- public CertificateSignRequest.Builder setScmID(String s) {
- this.scmID = s;
- return this;
- }
-
- // Support SAN extenion with DNS and RFC822 Name
- // other name type will be added as needed.
- public CertificateSignRequest.Builder addDnsName(String dnsName) {
- Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
- this.addAltName(GeneralName.dNSName, dnsName);
- return this;
- }
-
- public CertificateSignRequest.Builder addRfc822Name(String name) {
- Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
- this.addAltName(GeneralName.rfc822Name, name);
- return this;
- }
-
- // IP address is subject to change which is optional for now.
- public CertificateSignRequest.Builder addIpAddress(String ip) {
- Preconditions.checkNotNull(ip, "Ip address cannot be null");
- this.addAltName(GeneralName.iPAddress, ip);
- return this;
- }
-
- private CertificateSignRequest.Builder addAltName(int tag, String name) {
- if (altNames == null) {
- altNames = new ArrayList<>();
- }
- altNames.add(new GeneralName(tag, name));
- return this;
- }
-
- public CertificateSignRequest.Builder setCA(Boolean isCA) {
- this.ca = isCA;
- return this;
- }
-
- private Extension getKeyUsageExtension() throws IOException {
- int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
- | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
-
- if (ca) {
- keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
- }
- KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
- return new Extension(Extension.keyUsage, true,
- new DEROctetString(keyUsage));
- }
-
- private Optional<Extension> getSubjectAltNameExtension() throws
- IOException {
- if (altNames != null) {
- return Optional.of(new Extension(Extension.subjectAlternativeName,
- true, new DEROctetString(new GeneralNames(
- altNames.toArray(new GeneralName[altNames.size()])))));
- }
- return Optional.empty();
- }
-
- private Extension getBasicExtension() throws IOException {
- // We don't set pathLenConstraint means no limit is imposed.
- return new Extension(Extension.basicConstraints,
- true, new DEROctetString(new BasicConstraints(ca)));
- }
-
- private Extensions createExtensions() throws IOException {
- List<Extension> extensions = new ArrayList<>();
-
- // Add basic extension
- extensions.add(getBasicExtension());
-
- // Add key usage extension
- extensions.add(getKeyUsageExtension());
-
- // Add subject alternate name extension
- Optional<Extension> san = getSubjectAltNameExtension();
- if (san.isPresent()) {
- extensions.add(san.get());
- }
-
- return new Extensions(
- extensions.toArray(new Extension[extensions.size()]));
- }
-
- public PKCS10CertificationRequest build() throws SCMSecurityException {
- Preconditions.checkNotNull(key, "KeyPair cannot be null");
- Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
- "cannot be blank");
- Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
- "cannot be blank");
- Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
- "be blank");
-
- try {
- CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
- clusterID, key, config, createExtensions());
- return csr.generateCSR();
- } catch (IOException ioe) {
- throw new CertificateException(String.format("Unable to create " +
- "extension for certificate sign request for %s.", SecurityUtil
- .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
- } catch (OperatorCreationException ex) {
- throw new CertificateException(String.format("Unable to create " +
- "certificate sign request for %s.", SecurityUtil
- .getDistinguishedName(subject, scmID, clusterID)),
- ex.getCause());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
deleted file mode 100644
index 85fba9b..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.certificates;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.util.Time;
-import org.apache.logging.log4j.util.Strings;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.BasicConstraints;
-import org.bouncycastle.asn1.x509.Extension;
-import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
-import org.bouncycastle.cert.CertIOException;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v3CertificateBuilder;
-import org.bouncycastle.operator.ContentSigner;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-
-import java.math.BigInteger;
-import java.security.KeyPair;
-import java.time.Duration;
-import java.util.Date;
-
-/**
- * A Self Signed Certificate with CA basic constraint can be used to boot-strap
- * a certificate infra-structure, if no external certificate is provided.
- */
-public final class SelfSignedCertificate {
- private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
- private String subject;
- private String clusterID;
- private String scmID;
- private Date beginDate;
- private Date endDate;
- private KeyPair key;
- private SecurityConfig config;
- private boolean isCA;
-
- /**
- * Private Ctor invoked only via Builder Interface.
- * @param subject - Subject
- * @param scmID - SCM ID
- * @param clusterID - Cluster ID
- * @param beginDate - NotBefore
- * @param endDate - Not After
- * @param configuration - SCM Config
- * @param keyPair - KeyPair
- * @param ca - isCA?
- */
- private SelfSignedCertificate(String subject, String scmID, String clusterID,
- Date beginDate, Date endDate, SecurityConfig configuration,
- KeyPair keyPair, boolean ca) {
- this.subject = subject;
- this.clusterID = clusterID;
- this.scmID = scmID;
- this.beginDate = beginDate;
- this.endDate = endDate;
- config = configuration;
- this.key = keyPair;
- this.isCA = ca;
- }
-
- @VisibleForTesting
- public static String getNameFormat() {
- return NAME_FORMAT;
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- private X509CertificateHolder generateCertificate()
- throws OperatorCreationException, CertIOException {
- // For the Root Certificate we form the name from Subject, SCM ID and
- // Cluster ID.
- String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
- X500Name name = new X500Name(dnName);
- byte[] encoded = key.getPublic().getEncoded();
- SubjectPublicKeyInfo publicKeyInfo =
- SubjectPublicKeyInfo.getInstance(encoded);
-
-
- ContentSigner contentSigner =
- new JcaContentSignerBuilder(config.getSignatureAlgo())
- .setProvider(config.getProvider()).build(key.getPrivate());
-
- // Please note: Since this is a root certificate we use "ONE" as the
- // serial number. Also note that skip enforcing locale or UTC. We are
- // trying to operate at the Days level, hence Time zone is also skipped for
- // now.
- BigInteger serial = BigInteger.ONE;
- if (!isCA) {
- serial = new BigInteger(Long.toString(Time.monotonicNow()));
- }
-
- X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
- serial, beginDate, endDate, name, publicKeyInfo);
-
- if (isCA) {
- builder.addExtension(Extension.basicConstraints, true,
- new BasicConstraints(true));
- }
- return builder.build(contentSigner);
- }
-
- /**
- * Builder class for Root Certificates.
- */
- public static class Builder {
- private String subject;
- private String clusterID;
- private String scmID;
- private Date beginDate;
- private Date endDate;
- private KeyPair key;
- private SecurityConfig config;
- private boolean isCA;
-
- public Builder setConfiguration(Configuration configuration) {
- this.config = new SecurityConfig(configuration);
- return this;
- }
-
- public Builder setKey(KeyPair keyPair) {
- this.key = keyPair;
- return this;
- }
-
- public Builder setSubject(String subjectString) {
- this.subject = subjectString;
- return this;
- }
-
- public Builder setClusterID(String s) {
- this.clusterID = s;
- return this;
- }
-
- public Builder setScmID(String s) {
- this.scmID = s;
- return this;
- }
-
- public Builder setBeginDate(Date date) {
- this.beginDate = new Date(date.toInstant().toEpochMilli());
- return this;
- }
-
- public Builder setEndDate(Date date) {
- this.endDate = new Date(date.toInstant().toEpochMilli());
- return this;
- }
-
- public Builder makeCA() {
- isCA = true;
- return this;
- }
-
- public X509CertificateHolder build() throws SCMSecurityException {
- Preconditions.checkNotNull(key, "Key cannot be null");
- Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
- "cannot be blank");
- Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
- "cannot be blank");
- Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
- "be blank");
-
- Preconditions.checkArgument(beginDate.before(endDate), "Certificate " +
- "begin date should be before end date");
-
- Duration certDuration = Duration.between(beginDate.toInstant(),
- endDate.toInstant());
- Preconditions.checkArgument(
- certDuration.compareTo(config.getMaxCertificateDuration()) < 0,
- "Certificate life time cannot be greater than max configured value.");
-
-
- SelfSignedCertificate rootCertificate =
- new SelfSignedCertificate(this.subject,
- this.scmID, this.clusterID, this.beginDate, this.endDate,
- this.config, key, isCA);
- try {
- return rootCertificate.generateCertificate();
- } catch (OperatorCreationException | CertIOException e) {
- throw new CertificateException("Unable to create root certificate.",
- e.getCause());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
deleted file mode 100644
index e88737c..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Utils for Certificates.
- */
-package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
new file mode 100644
index 0000000..3624b32
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CertificateServer.
+ */
+public final class CertificateSignRequest {
+ private final KeyPair keyPair;
+ private final SecurityConfig config;
+ private final Extensions extensions;
+ private String subject;
+ private String clusterID;
+ private String scmID;
+
+ /**
+ * Private Ctor for CSR.
+ *
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param keyPair - KeyPair
+ * @param config - SCM Config
+ * @param extensions - CSR extensions
+ */
+ private CertificateSignRequest(String subject, String scmID, String clusterID,
+ KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.keyPair = keyPair;
+ this.config = config;
+ this.extensions = extensions;
+ }
+
+ private PKCS10CertificationRequest generateCSR() throws
+ OperatorCreationException {
+ X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+ clusterID);
+ PKCS10CertificationRequestBuilder p10Builder =
+ new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider())
+ .build(keyPair.getPrivate());
+
+ if (extensions != null) {
+ p10Builder.addAttribute(
+ PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+ }
+ return p10Builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Certificate Sign Request.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private KeyPair key;
+ private SecurityConfig config;
+ private List<GeneralName> altNames;
+ private Boolean ca = false;
+
+ public CertificateSignRequest.Builder setConfiguration(
+ Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ // Support SAN extenion with DNS and RFC822 Name
+ // other name type will be added as needed.
+ public CertificateSignRequest.Builder addDnsName(String dnsName) {
+ Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+ this.addAltName(GeneralName.dNSName, dnsName);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder addRfc822Name(String name) {
+ Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+ this.addAltName(GeneralName.rfc822Name, name);
+ return this;
+ }
+
+ // IP address is subject to change which is optional for now.
+ public CertificateSignRequest.Builder addIpAddress(String ip) {
+ Preconditions.checkNotNull(ip, "Ip address cannot be null");
+ this.addAltName(GeneralName.iPAddress, ip);
+ return this;
+ }
+
+ private CertificateSignRequest.Builder addAltName(int tag, String name) {
+ if (altNames == null) {
+ altNames = new ArrayList<>();
+ }
+ altNames.add(new GeneralName(tag, name));
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setCA(Boolean isCA) {
+ this.ca = isCA;
+ return this;
+ }
+
+ private Extension getKeyUsageExtension() throws IOException {
+ int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+ | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+ if (ca) {
+ keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+ }
+ KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+ return new Extension(Extension.keyUsage, true,
+ new DEROctetString(keyUsage));
+ }
+
+ private Optional<Extension> getSubjectAltNameExtension() throws
+ IOException {
+ if (altNames != null) {
+ return Optional.of(new Extension(Extension.subjectAlternativeName,
+ true, new DEROctetString(new GeneralNames(
+ altNames.toArray(new GeneralName[altNames.size()])))));
+ }
+ return Optional.empty();
+ }
+
+ private Extension getBasicExtension() throws IOException {
+ // We don't set pathLenConstraint means no limit is imposed.
+ return new Extension(Extension.basicConstraints,
+ true, new DEROctetString(new BasicConstraints(ca)));
+ }
+
+ private Extensions createExtensions() throws IOException {
+ List<Extension> extensions = new ArrayList<>();
+
+ // Add basic extension
+ extensions.add(getBasicExtension());
+
+ // Add key usage extension
+ extensions.add(getKeyUsageExtension());
+
+ // Add subject alternate name extension
+ Optional<Extension> san = getSubjectAltNameExtension();
+ if (san.isPresent()) {
+ extensions.add(san.get());
+ }
+
+ return new Extensions(
+ extensions.toArray(new Extension[extensions.size()]));
+ }
+
+ public PKCS10CertificationRequest build() throws SCMSecurityException {
+ Preconditions.checkNotNull(key, "KeyPair cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ try {
+ CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+ clusterID, key, config, createExtensions());
+ return csr.generateCSR();
+ } catch (IOException ioe) {
+ throw new CertificateException(String.format("Unable to create " +
+ "extension for certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+ } catch (OperatorCreationException ex) {
+ throw new CertificateException(String.format("Unable to create " +
+ "certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)),
+ ex.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
new file mode 100644
index 0000000..1fd6d7c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.util.Time;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+
+/**
+ * A Self Signed Certificate with CertificateServer basic constraint can be used
+ * to bootstrap a certificate infrastructure, if no external certificate is
+ * provided.
+ */
+public final class SelfSignedCertificate {
+ private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private LocalDate beginDate;
+ private LocalDate endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+
+ /**
+ * Private Ctor invoked only via Builder Interface.
+ *
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param beginDate - NotBefore
+ * @param endDate - Not After
+ * @param configuration - SCM Config
+ * @param keyPair - KeyPair
+ */
+ private SelfSignedCertificate(String subject, String scmID, String clusterID,
+ LocalDate beginDate, LocalDate endDate, SecurityConfig configuration,
+ KeyPair keyPair) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.beginDate = beginDate;
+ this.endDate = endDate;
+ config = configuration;
+ this.key = keyPair;
+ }
+
+ @VisibleForTesting
+ public static String getNameFormat() {
+ return NAME_FORMAT;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ private X509CertificateHolder generateCertificate(boolean isCA)
+ throws OperatorCreationException, IOException {
+ // For the Root Certificate we form the name from Subject, SCM ID and
+ // Cluster ID.
+ String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
+ X500Name name = new X500Name(dnName);
+ byte[] encoded = key.getPublic().getEncoded();
+ SubjectPublicKeyInfo publicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(encoded);
+
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider()).build(key.getPrivate());
+
+ // Please note: Since this is a root certificate we use "ONE" as the
+ // serial number. Also note that skip enforcing locale or UTC. We are
+ // trying to operate at the Days level, hence Time zone is also skipped for
+ // now.
+ BigInteger serial = BigInteger.ONE;
+ if (!isCA) {
+ serial = new BigInteger(Long.toString(Time.monotonicNow()));
+ }
+
+ ZoneOffset zoneOffset =
+ beginDate.atStartOfDay(ZoneOffset.systemDefault()).getOffset();
+
+ // Valid from the Start of the day when we generate this Certificate.
+ Date validFrom =
+ Date.from(beginDate.atTime(LocalTime.MIN).toInstant(zoneOffset));
+
+ // Valid till end day finishes.
+ Date validTill =
+ Date.from(endDate.atTime(LocalTime.MAX).toInstant(zoneOffset));
+
+ X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
+ serial, validFrom, validTill, name, publicKeyInfo);
+
+ if (isCA) {
+ builder.addExtension(Extension.basicConstraints, true,
+ new BasicConstraints(true));
+ int keyUsageFlag = KeyUsage.keyCertSign | KeyUsage.cRLSign;
+ KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+ builder.addExtension(Extension.keyUsage, false,
+ new DEROctetString(keyUsage));
+ }
+ return builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Root Certificates.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private LocalDate beginDate;
+ private LocalDate endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+ private boolean isCA;
+
+ public Builder setConfiguration(Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ public Builder setBeginDate(LocalDate date) {
+ this.beginDate = date;
+ return this;
+ }
+
+ public Builder setEndDate(LocalDate date) {
+ this.endDate = date;
+ return this;
+ }
+
+ public Builder makeCA() {
+ isCA = true;
+ return this;
+ }
+
+ public X509CertificateHolder build()
+ throws SCMSecurityException, IOException {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ Preconditions.checkArgument(beginDate.isBefore(endDate), "Certificate " +
+ "begin date should be before end date");
+
+ // We just read the beginDate and EndDate as Start of the Day and
+ // confirm that we do not violate the maxDuration Config.
+ Duration certDuration = Duration.between(beginDate.atStartOfDay(),
+ endDate.atStartOfDay());
+ Duration maxDuration = config.getMaxCertificateDuration();
+ if (certDuration.compareTo(maxDuration) > 0) {
+ throw new SCMSecurityException("The cert duration violates the " +
+ "maximum configured value. Please check the hdds.x509.max" +
+ ".duration config key. Current Value: " + certDuration +
+ " config: " + maxDuration);
+ }
+
+ SelfSignedCertificate rootCertificate =
+ new SelfSignedCertificate(this.subject,
+ this.scmID, this.clusterID, this.beginDate, this.endDate,
+ this.config, key);
+ try {
+ return rootCertificate.generateCertificate(isCA);
+ } catch (OperatorCreationException | CertIOException e) {
+ throw new CertificateException("Unable to create root certificate.",
+ e.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
new file mode 100644
index 0000000..945adc9
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Utils for Certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates.utils;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-8974. Improve the assertion
message in TestGPUResourceHandler. (Zhankun Tang via wangda)
Posted by xy...@apache.org.
YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda)
Change-Id: I4eb58e9d251d5f54e7feffc4fbb813b4f5ae4b1b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ebeda98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ebeda98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ebeda98
Branch: refs/heads/HDDS-4
Commit: 8ebeda98a9d3eac45598a33bae3d62e3ebb92cad
Parents: 9ed8756
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:36:30 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:36:30 2018 -0800
----------------------------------------------------------------------
.../linux/resources/gpu/TestGpuResourceHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ebeda98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 10e5cd1..18785e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -465,7 +465,7 @@ public class TestGpuResourceHandler {
caughtException = true;
}
Assert.assertTrue(
- "Should fail since requested device Id is not in allowed list",
+ "Should fail since requested device Id is already assigned",
caughtException);
// Make sure internal state not changed.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDDS-850. ReadStateMachineData
hits OverlappingFileLockException in ContainerStateMachine. Contributed by
Shashikant Banerjee.
Posted by xy...@apache.org.
HDDS-850. ReadStateMachineData hits OverlappingFileLockException in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e102f9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e102f9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e102f9a
Branch: refs/heads/HDDS-4
Commit: 5e102f9aa54d3057ef5f0755d45428f22a24990b
Parents: 7eb0d3a
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Thu Nov 29 22:20:08 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Thu Nov 29 22:20:08 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 ++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 9 ++
.../main/proto/DatanodeContainerProtocol.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 8 ++
.../server/ratis/ContainerStateMachine.java | 134 +++++++++++--------
.../server/ratis/XceiverServerRatis.java | 14 +-
.../container/keyvalue/KeyValueHandler.java | 7 +-
.../keyvalue/impl/ChunkManagerImpl.java | 11 +-
.../keyvalue/interfaces/ChunkManager.java | 5 +-
.../keyvalue/TestChunkManagerImpl.java | 6 +-
.../common/impl/TestContainerPersistence.java | 11 +-
11 files changed, 143 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 6733b8e..062b101 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -93,6 +93,14 @@ public final class ScmConfigKeys {
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE =
"dfs.container.ratis.log.queue.size";
public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT = 128;
+
+ // expiry interval stateMachineData cache entry inside containerStateMachine
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+ "dfs.container.ratis.statemachine.cache.expiry.interval";
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+ "10s";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 879f773..df233f7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -249,6 +249,15 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+ ScmConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+ ScmConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT;
+
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 3695b6b..5237af8 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -392,6 +392,7 @@ message WriteChunkResponseProto {
message ReadChunkRequestProto {
required DatanodeBlockID blockID = 1;
required ChunkInfo chunkData = 2;
+ optional bool readFromTmpFile = 3 [default = false];
}
message ReadChunkResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aa22b2b..0545805 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -190,6 +190,14 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.statemachine.cache.expiry.interval</name>
+ <value>10s</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>The interval till which the stateMachine data in ratis
+ will be cached inside the ContainerStateMachine.
+ </description>
+ </property>
+ <property>
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 24ecbb2..2e0c70e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -27,6 +29,7 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerConstants;
+import org.apache.ratis.server.impl.RaftServerProxy;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
import org.apache.ratis.thirdparty.com.google.protobuf
@@ -60,12 +63,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
@@ -116,12 +123,11 @@ public class ContainerStateMachine extends BaseStateMachine {
private final XceiverServerRatis ratisServer;
private final ConcurrentHashMap<Long, CompletableFuture<Message>>
writeChunkFutureMap;
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- createContainerFutureMap;
private ExecutorService[] executors;
private final int numExecutors;
private final Map<Long, Long> applyTransactionCompletionMap;
private long lastIndex;
+ private final Cache<Long, ByteString> stateMachineDataCache;
/**
* CSM metrics.
*/
@@ -129,7 +135,7 @@ public class ContainerStateMachine extends BaseStateMachine {
public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher,
ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer,
- List<ExecutorService> executors) {
+ List<ExecutorService> executors, long expiryInterval) {
this.gid = gid;
this.dispatcher = dispatcher;
this.chunkExecutor = chunkExecutor;
@@ -138,9 +144,13 @@ public class ContainerStateMachine extends BaseStateMachine {
this.numExecutors = executors.size();
this.executors = executors.toArray(new ExecutorService[numExecutors]);
this.writeChunkFutureMap = new ConcurrentHashMap<>();
- this.createContainerFutureMap = new ConcurrentHashMap<>();
applyTransactionCompletionMap = new ConcurrentHashMap<>();
this.lastIndex = RaftServerConstants.INVALID_LOG_INDEX;
+ stateMachineDataCache = CacheBuilder.newBuilder()
+ .expireAfterAccess(expiryInterval, TimeUnit.MILLISECONDS)
+ // set the limit on no of cached entries equal to no of max threads
+ // executing writeStateMachineData
+ .maximumSize(chunkExecutor.getCorePoolSize()).build();
}
@Override
@@ -257,14 +267,6 @@ public class ContainerStateMachine extends BaseStateMachine {
.setStateMachineData(dataContainerCommandProto.toByteString())
.setLogData(commitContainerCommandProto.toByteString())
.build();
- } else if (proto.getCmdType() == Type.CreateContainer) {
- return TransactionContext.newBuilder()
- .setClientRequest(request)
- .setStateMachine(this)
- .setServerRole(RaftPeerRole.LEADER)
- .setStateMachineData(request.getMessage().getContent())
- .setLogData(request.getMessage().getContent())
- .build();
} else {
return TransactionContext.newBuilder()
.setClientRequest(request)
@@ -310,17 +312,17 @@ public class ContainerStateMachine extends BaseStateMachine {
private CompletableFuture<Message> handleWriteChunk(
ContainerCommandRequestProto requestProto, long entryIndex) {
final WriteChunkRequestProto write = requestProto.getWriteChunk();
- long containerID = write.getBlockID().getContainerID();
- CompletableFuture<Message> future =
- createContainerFutureMap.get(containerID);
- CompletableFuture<Message> writeChunkFuture;
- if (future != null) {
- writeChunkFuture = future.thenApplyAsync(
- v -> runCommand(requestProto), chunkExecutor);
- } else {
- writeChunkFuture = CompletableFuture.supplyAsync(
- () -> runCommand(requestProto), chunkExecutor);
+ RaftServer server = ratisServer.getServer();
+ Preconditions.checkState(server instanceof RaftServerProxy);
+ try {
+ if (((RaftServerProxy) server).getImpl(gid).isLeader()) {
+ stateMachineDataCache.put(entryIndex, write.getData());
+ }
+ } catch (IOException ioe) {
+ return completeExceptionally(ioe);
}
+ CompletableFuture<Message> writeChunkFuture = CompletableFuture
+ .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
writeChunkFutureMap.put(entryIndex, writeChunkFuture);
LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+ " logIndex " + entryIndex + " chunkName " + write.getChunkData()
@@ -337,14 +339,6 @@ public class ContainerStateMachine extends BaseStateMachine {
return writeChunkFuture;
}
- private CompletableFuture<Message> handleCreateContainer(
- ContainerCommandRequestProto requestProto) {
- long containerID = requestProto.getContainerID();
- createContainerFutureMap.
- computeIfAbsent(containerID, k -> new CompletableFuture<>());
- return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
- }
-
/*
* writeStateMachineData calls are not synchronized with each other
* and also with applyTransaction.
@@ -356,9 +350,10 @@ public class ContainerStateMachine extends BaseStateMachine {
final ContainerCommandRequestProto requestProto =
getRequestProto(getStateMachineData(entry.getStateMachineLogEntry()));
Type cmdType = requestProto.getCmdType();
+
+ // For only writeChunk, there will be writeStateMachineData call.
+ // CreateContainer will happen as a part of writeChunk only.
switch (cmdType) {
- case CreateContainer:
- return handleCreateContainer(requestProto);
case WriteChunk:
return handleWriteChunk(requestProto, entry.getIndex());
default:
@@ -397,7 +392,10 @@ public class ContainerStateMachine extends BaseStateMachine {
ReadChunkRequestProto.Builder readChunkRequestProto =
ReadChunkRequestProto.newBuilder()
.setBlockID(writeChunkRequestProto.getBlockID())
- .setChunkData(writeChunkRequestProto.getChunkData());
+ .setChunkData(writeChunkRequestProto.getChunkData())
+ // set readFromTempFile to true in case, the chunkFile does
+ // not exist as applyTransaction is not executed for this entry yet.
+ .setReadFromTmpFile(true);
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto.newBuilder(requestProto)
.setCmdType(Type.ReadChunk)
@@ -409,15 +407,39 @@ public class ContainerStateMachine extends BaseStateMachine {
dispatchCommand(dataContainerCommandProto);
ReadChunkResponseProto responseProto = response.getReadChunk();
+ ByteString data = responseProto.getData();
// assert that the response has data in it.
- Preconditions.checkNotNull(responseProto.getData());
+ Preconditions.checkNotNull(data);
+ return data;
+ }
+ /**
+ * Reads the Entry from the Cache or loads it back by reading from disk.
+ */
+ private ByteString getCachedStateMachineData(Long logIndex,
+ ContainerCommandRequestProto requestProto) throws ExecutionException {
+ try {
+ return reconstructWriteChunkRequest(
+ stateMachineDataCache.get(logIndex, new Callable<ByteString>() {
+ @Override
+ public ByteString call() throws Exception {
+ return readStateMachineData(requestProto);
+ }
+ }), requestProto);
+ } catch (ExecutionException e) {
+ throw e;
+ }
+ }
+
+ private ByteString reconstructWriteChunkRequest(ByteString data,
+ ContainerCommandRequestProto requestProto) {
+ WriteChunkRequestProto writeChunkRequestProto =
+ requestProto.getWriteChunk();
// reconstruct the write chunk request
final WriteChunkRequestProto.Builder dataWriteChunkProto =
WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
// adding the state machine data
- .setData(responseProto.getData())
- .setStage(Stage.WRITE_DATA);
+ .setData(data).setStage(Stage.WRITE_DATA);
ContainerCommandRequestProto.Builder newStateMachineProto =
ContainerCommandRequestProto.newBuilder(requestProto)
@@ -455,18 +477,21 @@ public class ContainerStateMachine extends BaseStateMachine {
if (!getStateMachineData(smLogEntryProto).isEmpty()) {
return CompletableFuture.completedFuture(ByteString.EMPTY);
}
-
try {
final ContainerCommandRequestProto requestProto =
getRequestProto(entry.getStateMachineLogEntry().getLogData());
// readStateMachineData should only be called for "write" to Ratis.
Preconditions.checkArgument(!HddsUtils.isReadOnly(requestProto));
-
if (requestProto.getCmdType() == Type.WriteChunk) {
- return CompletableFuture.supplyAsync(() ->
- readStateMachineData(requestProto), chunkExecutor);
- } else if (requestProto.getCmdType() == Type.CreateContainer) {
- return CompletableFuture.completedFuture(requestProto.toByteString());
+ CompletableFuture<ByteString> future = new CompletableFuture<>();
+ return future.supplyAsync(() -> {
+ try {
+ return getCachedStateMachineData(entry.getIndex(), requestProto);
+ } catch (ExecutionException e) {
+ future.completeExceptionally(e);
+ return null;
+ }
+ }, chunkExecutor);
} else {
throw new IllegalStateException("Cmd type:" + requestProto.getCmdType()
+ " cannot have state machine data");
@@ -559,19 +584,6 @@ public class ContainerStateMachine extends BaseStateMachine {
future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
getCommandExecutor(requestProto));
}
- // Mark the createContainerFuture complete so that writeStateMachineData
- // for WriteChunk gets unblocked
- if (cmdType == Type.CreateContainer) {
- long containerID = requestProto.getContainerID();
- future.thenApply(
- r -> {
- createContainerFutureMap.remove(containerID).complete(null);
- LOG.info("create Container Transaction completed for container " +
- containerID + " log index " + index);
- return r;
- });
- }
-
lastIndex = index;
future.thenAccept(m -> {
final Long previous =
@@ -593,6 +605,11 @@ public class ContainerStateMachine extends BaseStateMachine {
return future;
}
+ private void evictStateMachineCache() {
+ stateMachineDataCache.invalidateAll();
+ stateMachineDataCache.cleanUp();
+ }
+
@Override
public void notifySlowness(RaftGroup group, RoleInfoProto roleInfoProto) {
ratisServer.handleNodeSlowness(group, roleInfoProto);
@@ -605,6 +622,13 @@ public class ContainerStateMachine extends BaseStateMachine {
}
@Override
+ public void notifyNotLeader(Collection<TransactionContext> pendingEntries)
+ throws IOException {
+ evictStateMachineCache();
+ }
+
+ @Override
public void close() throws IOException {
+ evictStateMachineCache();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a4927e0..2675483 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -105,6 +105,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final StateContext context;
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
+ private final long cacheEntryExpiryInteval;
+
private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
@@ -128,6 +130,11 @@ public final class XceiverServerRatis implements XceiverServerSpi {
conf.getEnum(OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT);
this.executors = new ArrayList<>();
+ cacheEntryExpiryInteval = conf.getTimeDuration(OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
this.dispatcher = dispatcher;
for (int i = 0; i < numContainerOpExecutors; i++) {
executors.add(Executors.newSingleThreadExecutor());
@@ -141,8 +148,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
}
private ContainerStateMachine getStateMachine(RaftGroupId gid) {
- return new ContainerStateMachine(gid, dispatcher, chunkExecutor,
- this, Collections.unmodifiableList(executors));
+ return new ContainerStateMachine(gid, dispatcher, chunkExecutor, this,
+ Collections.unmodifiableList(executors), cacheEntryExpiryInteval);
}
private RaftProperties newRaftProperties(Configuration conf) {
@@ -304,6 +311,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties,
numSyncRetries);
+ // Enable the StateMachineCaching
+ RaftServerConfigKeys.Log.StateMachineData
+ .setCachingEnabled(properties, true);
return properties;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 36c48c5..a16129e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -531,7 +531,8 @@ public class KeyValueHandler extends Handler {
.getChunkData());
Preconditions.checkNotNull(chunkInfo);
- data = chunkManager.readChunk(kvContainer, blockID, chunkInfo);
+ data = chunkManager.readChunk(kvContainer, blockID, chunkInfo,
+ request.getReadChunk().getReadFromTmpFile());
metrics.incContainerBytesStats(Type.ReadChunk, data.length);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -702,8 +703,10 @@ public class KeyValueHandler extends Handler {
ContainerProtos.ChunkInfo chunkInfo = null;
ByteString dataBuf = ByteString.EMPTY;
for (ContainerProtos.ChunkInfo chunk : responseData.getChunks()) {
+ // if the block is committed, all chunks must have been committed.
+ // Tmp chunk files won't exist here.
byte[] data = chunkManager.readChunk(kvContainer, blockID,
- ChunkInfo.getFromProtoBuf(chunk));
+ ChunkInfo.getFromProtoBuf(chunk), false);
ByteString current = ByteString.copyFrom(data);
dataBuf = dataBuf.concat(current);
chunkInfo = chunk;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index a1c3d01..e41346c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -173,13 +173,14 @@ public class ChunkManagerImpl implements ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
+ * @param readFromTmpFile whether to read from tmp chunk file or not.
* @return byte array
* @throws StorageContainerException
* TODO: Right now we do not support partial reads and writes of chunks.
* TODO: Explore if we need to do that for ozone.
*/
- public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info)
- throws StorageContainerException {
+ public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+ boolean readFromTmpFile) throws StorageContainerException {
try {
KeyValueContainerData containerData = (KeyValueContainerData) container
.getContainerData();
@@ -194,6 +195,12 @@ public class ChunkManagerImpl implements ChunkManager {
if (containerData.getLayOutVersion() == ChunkLayOutVersion
.getLatestVersion().getVersion()) {
File chunkFile = ChunkUtils.getChunkFile(containerData, info);
+
+ // In case the chunk file does not exist but tmp chunk file exist,
+ // read from tmp chunk file if readFromTmpFile is set to true
+ if (!chunkFile.exists() && readFromTmpFile) {
+ chunkFile = getTmpChunkFile(chunkFile, info);
+ }
data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
containerData.incrReadCount();
long length = chunkFile.length();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 3b06585..f90346e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -51,14 +51,15 @@ public interface ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
+ * @param readFromTmpFile whether to read from tmp chunk file or not
* @return byte array
* @throws StorageContainerException
*
* TODO: Right now we do not support partial reads and writes of chunks.
* TODO: Explore if we need to do that for ozone.
*/
- byte[] readChunk(Container container, BlockID blockID, ChunkInfo info) throws
- StorageContainerException;
+ byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+ boolean readFromTmpFile) throws StorageContainerException;
/**
* Deletes a given chunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 717dd05..ef77204 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -189,7 +189,7 @@ public class TestChunkManagerImpl {
checkWriteIOStats(data.length, 1);
checkReadIOStats(0, 0);
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo);
+ chunkInfo, false);
assertEquals(expectedData.length, data.length);
assertTrue(Arrays.equals(expectedData, data));
checkReadIOStats(data.length, 1);
@@ -226,7 +226,7 @@ public class TestChunkManagerImpl {
try {
// trying to read a chunk, where chunk file does not exist
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo);
+ chunkInfo, false);
fail("testReadChunkFileNotExists failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
@@ -249,7 +249,7 @@ public class TestChunkManagerImpl {
for (int i=0; i<100; i++) {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
- chunkManager.readChunk(keyValueContainer, blockID, chunkInfo);
+ chunkManager.readChunk(keyValueContainer, blockID, chunkInfo, false);
}
checkReadIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index fb7b0c5..5d19a10 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -406,7 +406,7 @@ public class TestContainerPersistence {
for (int x = 0; x < chunkCount; x++) {
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
ChunkInfo info = fileHashMap.get(fileName);
- byte[] data = chunkManager.readChunk(container, blockID, info);
+ byte[] data = chunkManager.readChunk(container, blockID, info, false);
ChecksumData checksumData = checksum.computeChecksum(data);
Assert.assertEquals(info.getChecksumData(), checksumData);
}
@@ -435,11 +435,11 @@ public class TestContainerPersistence {
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
COMBINED);
- byte[] readData = chunkManager.readChunk(container, blockID, info);
+ byte[] readData = chunkManager.readChunk(container, blockID, info, false);
assertTrue(Arrays.equals(data, readData));
ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length);
- byte[] readData2 = chunkManager.readChunk(container, blockID, info2);
+ byte[] readData2 = chunkManager.readChunk(container, blockID, info2, false);
assertEquals(length, readData2.length);
assertTrue(Arrays.equals(
Arrays.copyOfRange(data, start, start + length), readData2));
@@ -513,7 +513,8 @@ public class TestContainerPersistence {
// Request to read the whole data in a single go.
ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0,
datalen * chunkCount);
- byte[] newdata = chunkManager.readChunk(container, blockID, largeChunk);
+ byte[] newdata =
+ chunkManager.readChunk(container, blockID, largeChunk, false);
MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
newSha.update(newdata);
Assert.assertEquals(Hex.encodeHexString(oldSha.digest()),
@@ -543,7 +544,7 @@ public class TestContainerPersistence {
chunkManager.deleteChunk(container, blockID, info);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the chunk file.");
- chunkManager.readChunk(container, blockID, info);
+ chunkManager.readChunk(container, blockID, info, false);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager
Delegation Token support. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 904e597..c1695f1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -17,16 +17,23 @@
*/
package org.apache.hadoop.ozone;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.slf4j.event.Level.INFO;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.PrivilegedExceptionAction;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Callable;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,14 +42,29 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyPEMWriter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.LambdaTestUtils;
@@ -52,6 +74,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +84,8 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public final class TestSecureOzoneCluster {
+ private static final String TEST_USER = "testUgiUser";
+ private static final int CLIENT_TIMEOUT = 2 * 1000;
private Logger LOGGER = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
@@ -81,14 +106,24 @@ public final class TestSecureOzoneCluster {
private static String clusterId;
private static String scmId;
private static String omId;
+ private OzoneManagerProtocolClientSideTranslatorPB omClient;
+ private KeyPair keyPair;
+ private Path metaDirPath;
@Before
public void init() {
try {
conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ metaDirPath = Paths.get(path, "om-meta");
+ conf.set(OZONE_METADATA_DIRS, metaDirPath.toString());
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
+ generateKeyPair(conf);
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -106,6 +141,10 @@ public final class TestSecureOzoneCluster {
if (om != null) {
om.stop();
}
+ if (omClient != null) {
+ omClient.close();
+ }
+ FileUtils.deleteQuietly(metaDirPath.toFile());
} catch (Exception e) {
LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
}
@@ -115,11 +154,11 @@ public final class TestSecureOzoneCluster {
throws Exception {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys
- .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
- conf.get(OMConfigKeys
- .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
+ conf.get(OMConfigKeys
+ .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
createPrincipal(omKeyTab,
conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
@@ -137,13 +176,14 @@ public final class TestSecureOzoneCluster {
miniKdc.start();
}
- private void stopMiniKdc() throws Exception {
+ private void stopMiniKdc() {
miniKdc.stop();
}
private void setSecureConfig(Configuration conf) throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- String host = KerberosUtil.getLocalHostName();
+ conf.setBoolean(OZONE_ENABLED, true);
+ String host = InetAddress.getLocalHost().getCanonicalHostName();
String realm = miniKdc.getRealm();
curUser = UserGroupInformation.getCurrentUser()
.getUserName();
@@ -195,7 +235,7 @@ public final class TestSecureOzoneCluster {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
scmStore.setClusterId(clusterId);
@@ -245,60 +285,261 @@ public final class TestSecureOzoneCluster {
}
/**
- * Tests the secure KSM Initialization Failure.
+ * Tests the secure om Initialization Failure.
*
* @throws IOException
*/
@Test
- public void testSecureKsmInitializationFailure() throws Exception {
+ public void testSecureOMInitializationFailure() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- OMStorage ksmStore = new OMStorage(conf);
- ksmStore.setClusterId("testClusterId");
- ksmStore.setScmId("testScmId");
- // writes the version file properties
- ksmStore.initialize();
+ setupOm(conf);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"non-existent-user@EXAMPLE.com");
testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
}
/**
- * Tests the secure KSM Initialization success.
+ * Tests the secure om Initialization success.
*
* @throws IOException
*/
@Test
- public void testSecureKsmInitializationSuccess() throws Exception {
+ public void testSecureOmInitializationSuccess() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+ GenericTestUtils.setLogLevel(OzoneManager.LOG, INFO);
+
+ setupOm(conf);
+ try {
+ om.start();
+ } catch (Exception ex) {
+ // Expects timeout failure from scmClient in om but om user login via
+ // kerberos should succeed.
+ Assert.assertTrue(logs.getOutput().contains("Ozone Manager login"
+ + " successful"));
+ }
+ }
+
+ /**
+ * Performs following tests for delegation token.
+ * 1. Get valid delegation token
+ * 2. Test successful token renewal.
+ * 3. Client can authenticate using token.
+ * 4. Delegation token renewal without Kerberos auth fails.
+ * 5. Test success of token cancellation.
+ * 5. Test failure of token cancellation.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDelegationToken() throws Exception {
+
+ // Capture logs for assertions
+ LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
GenericTestUtils
- .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
- org.slf4j.event.Level.INFO);
+ .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+ // Setup secure OM for start
+ setupOm(conf);
+ long omVersion =
+ RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+ // Start OM
+ om.start();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String username = ugi.getUserName();
+ ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+
+ // Get first OM client which will authenticate via Kerberos
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ CLIENT_TIMEOUT));
+
+ // Assert if auth was successful via Kerberos
+ Assert.assertFalse(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:KERBEROS)"));
+
+ // Case 1: Test successful delegation token.
+ Token<OzoneTokenIdentifier> token = omClient
+ .getDelegationToken(new Text("om"));
+
+ // Case 2: Test successful token renewal.
+ long renewalTime = omClient.renewDelegationToken(token);
+ Assert.assertTrue(renewalTime > 0);
+
+ // Check if token is of right kind and renewer is running om instance
+ Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+ Assert.assertEquals(token.getService().toString(),
+ OmUtils.getOmRpcAddress(conf));
+ omClient.close();
+
+ // Create a remote ugi and set its authentication method to Token
+ UserGroupInformation testUser = UserGroupInformation
+ .createRemoteUser(TEST_USER);
+ testUser.addToken(token);
+ testUser.setAuthenticationMethod(AuthMethod.TOKEN);
+ UserGroupInformation.setLoginUser(testUser);
+
+ // Get Om client, this time authentication should happen via Token
+ testUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), testUser, conf,
+ NetUtils.getDefaultSocketFactory(conf), CLIENT_TIMEOUT));
+ return null;
+ }
+ });
+
+ // Case 3: Test Client can authenticate using token.
+ Assert.assertFalse(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+ LambdaTestUtils.intercept(IOException.class, "Delete Volume failed," +
+ " error:VOLUME_NOT_FOUND",
+ () -> omClient.deleteVolume("vol1"));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+
+ // Case 4: Test failure of token renewal.
+ // Call to renewDelegationToken will fail but it will confirm that
+ // initial connection via DT succeeded
+ LambdaTestUtils.intercept(RemoteException.class, "Delegation "
+ + "Token can be renewed only with kerberos or web authentication",
+ () -> omClient.renewDelegationToken(token));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+ //testUser.setAuthenticationMethod(AuthMethod.KERBEROS);
+ UserGroupInformation.setLoginUser(ugi);
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ Client.getRpcTimeout(conf)));
+
+ // Case 5: Test success of token cancellation.
+ omClient.cancelDelegationToken(token);
+ omClient.close();
+
+ // Wait for client to timeout
+ Thread.sleep(CLIENT_TIMEOUT);
+
+ Assert.assertFalse(logs.getOutput().contains("Auth failed for"));
+
+ // Case 6: Test failure of token cancellation.
+ // Get Om client, this time authentication using Token will fail as
+ // token is expired
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), testUser, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ Client.getRpcTimeout(conf)));
+ LambdaTestUtils.intercept(RemoteException.class, "can't be found in cache",
+ () -> omClient.cancelDelegationToken(token));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth failed for"));
+ }
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- Path metaDirPath = Paths.get(path, "om-meta");
+ private void generateKeyPair(OzoneConfiguration config) throws Exception {
+ HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf);
+ keyPair = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(config);
+ pemWriter.writeKey(keyPair, true);
+ }
- OMStorage omStore = new OMStorage(conf);
+ /**
+ * Tests delegation token renewal.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDelegationTokenRenewal() throws Exception {
+ // Capture logs for assertions.
+ LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
+ GenericTestUtils
+ .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+ // Setup secure OM for start.
+ OzoneConfiguration newConf = new OzoneConfiguration(conf);
+ newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 500);
+ setupOm(newConf);
+ long omVersion =
+ RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+ OzoneManager.setTestSecureOmFlag(true);
+ // Start OM
+
+ om.start();
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String username = ugi.getUserName();
+
+ // Get first OM client which will authenticate via Kerberos
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ CLIENT_TIMEOUT));
+
+ // Since client is already connected get a delegation token
+ Token<OzoneTokenIdentifier> token = omClient
+ .getDelegationToken(new Text("om"));
+
+ // Check if token is of right kind and renewer is running om instance
+ Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+ Assert.assertEquals(token.getService().toString(),
+ OmUtils.getOmRpcAddress(conf));
+
+ // Renew delegation token
+ long expiryTime = omClient.renewDelegationToken(token);
+ Assert.assertTrue(expiryTime > 0);
+
+ // Test failure of delegation renewal
+ // 1. When renewer doesn't match (implicitly covers when renewer is
+ // null or empty )
+ Token token2 = omClient.getDelegationToken(new Text("randomService"));
+ LambdaTestUtils.intercept(RemoteException.class,
+ " with non-matching renewer randomService",
+ () -> omClient.renewDelegationToken(token2));
+
+ // 2. Test tampered token
+ OzoneTokenIdentifier tokenId = OzoneTokenIdentifier
+ .readProtoBuf(token.getIdentifier());
+ tokenId.setRenewer(new Text("om"));
+ tokenId.setMaxDate(System.currentTimeMillis() * 2);
+ Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
+ tokenId.getBytes(), token2.getPassword(), token2.getKind(),
+ token2.getService());
+ LambdaTestUtils
+ .intercept(RemoteException.class, "can't be found in cache",
+ () -> omClient.renewDelegationToken(tamperedToken));
+
+ // 3. When token maxExpiryTime exceeds
+ Thread.sleep(500);
+ LambdaTestUtils
+ .intercept(RemoteException.class, "om tried to renew an expired"
+ + " token", () -> omClient.renewDelegationToken(token));
+ }
+
+ private void setupOm(OzoneConfiguration config) throws Exception {
+ OMStorage omStore = new OMStorage(config);
omStore.setClusterId("testClusterId");
omStore.setScmId("testScmId");
// writes the version file properties
omStore.initialize();
- try {
- om = OzoneManager.createOm(null, conf);
- } catch (Exception ex) {
- // Expects timeout failure from scmClient in KSM but KSM user login via
- // kerberos should succeed
- Assert.assertTrue(
- logs.getOutput().contains("Ozone Manager login successful."));
- }
+ OzoneManager.setTestSecureOmFlag(true);
+ om = OzoneManager.createOm(null, config);
+ CertificateClient certClient = Mockito.mock(CertificateClient.class);
+ Mockito.when(certClient.getPrivateKey("om"))
+ .thenReturn(keyPair.getPrivate());
+ Mockito.when(certClient.getPublicKey("om"))
+ .thenReturn(keyPair.getPublic());
+ om.setCertClient(certClient);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a5cec20..c6616aa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
+import java.security.KeyPair;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -35,12 +37,20 @@ import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideT
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.security.OzoneSecurityException;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.security.OzoneSecretManager;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
@@ -53,7 +63,6 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -74,6 +83,8 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -99,6 +110,7 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.ozone.security.OzoneSecurityException.ResultCodes.*;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
@@ -134,13 +146,21 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
public static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
- private static final AuditLogger AUDIT =
- new AuditLogger(AuditLoggerType.OMLOGGER);
+ private static final AuditLogger AUDIT = new AuditLogger(
+ AuditLoggerType.OMLOGGER);
private static final String USAGE =
"Usage: \n ozone om [genericOptions] " + "[ "
+ StartupOption.INIT.getName() + " ]\n " + "ozone om [ "
+ StartupOption.HELP.getName() + " ]\n";
+ private static final String OM_DAEMON = "om";
+ private static boolean securityEnabled = false;
+ private static OzoneSecretManager secretManager;
+ // TO DO: For testing purpose only, remove before commiting
+ private KeyPair keyPair;
+ private CertificateClient certClient;
+ private static boolean testSecureOmFlag = false;
+ private final Text omRpcAddressTxt;
private final OzoneConfiguration configuration;
private RPC.Server omRpcServer;
private InetSocketAddress omRpcAddress;
@@ -174,21 +194,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
ResultCodes.OM_NOT_INITIALIZED);
}
- scmContainerClient = getScmContainerClient(configuration);
-
- // verifies that the SCM info in the OM Version file is correct.
- scmBlockClient = getScmBlockClient(configuration);
-
- ScmInfo scmInfo = scmBlockClient.getScmInfo();
- if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
- .getScmId().equals(omStorage.getScmId()))) {
- throw new OMException("SCM version info mismatch.",
- ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+ if (!testSecureOmFlag) {
+ scmContainerClient = getScmContainerClient(configuration);
+ // verifies that the SCM info in the OM Version file is correct.
+ scmBlockClient = getScmBlockClient(configuration);
+ ScmInfo scmInfo = scmBlockClient.getScmInfo();
+ if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
+ .getScmId().equals(omStorage.getScmId()))) {
+ throw new OMException("SCM version info mismatch.",
+ ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+ }
+ } else {
+ // For testing purpose only
+ scmContainerClient = null;
+ scmBlockClient = null;
}
+ final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
+ OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
+ BlockingService omService = newReflectiveBlockingService(
+ new OzoneManagerProtocolServerSideTranslatorPB(this));
+ final InetSocketAddress omNodeRpcAddr =
+ getOmAddress(configuration);
+ omRpcAddressTxt = new Text(OmUtils.getOmRpcAddress(configuration));
+ secretManager = createSecretManager(configuration);
+
+ omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
+ OzoneManagerProtocolPB.class, omService,
+ handlerCount);
+ omRpcAddress = updateRPCListenAddress(configuration,
+ OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
metadataManager = new OmMetadataManagerImpl(configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager);
@@ -258,6 +296,65 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
+ private OzoneSecretManager createSecretManager(
+ OzoneConfiguration conf)
+ throws IOException {
+ long tokenRemoverScanInterval =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY,
+ OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long tokenMaxLifetime =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+ OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long tokenRenewInterval =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ return new OzoneSecretManager(conf, tokenMaxLifetime, tokenRenewInterval,
+ tokenRemoverScanInterval, omRpcAddressTxt);
+ }
+
+ private void stopSecretManager() throws IOException {
+ if (secretManager != null) {
+ LOG.info("Stopping OM secret manager");
+ secretManager.stop();
+ }
+ }
+
+ private void startSecretManager() {
+ if (secretManager != null) {
+ try {
+ readKeyPair();
+ LOG.info("Starting OM secret manager");
+ secretManager.startThreads(keyPair);
+ } catch (IOException e) {
+ // Inability to start secret manager
+ // can't be recovered from.
+ LOG.error("Error starting secret manager.", e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void setCertClient(CertificateClient certClient) {
+ // TODO: Initialize it in contructor with implementation for certClient.
+ this.certClient = certClient;
+ }
+
+ /**
+ * Read private key from file.
+ */
+ private void readKeyPair() throws OzoneSecurityException {
+ try {
+ keyPair = new KeyPair(certClient.getPublicKey(OM_DAEMON),
+ certClient.getPrivateKey(OM_DAEMON));
+ } catch (Exception e) {
+ throw new OzoneSecurityException("Error reading private file for "
+ + "OzoneManager", e, OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
+ }
+ }
+
/**
* Login OM service user if security and Kerberos are enabled.
*
@@ -267,8 +364,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private static void loginOMUser(OzoneConfiguration conf)
throws IOException, AuthenticationException {
- if (SecurityUtil.getAuthenticationMethod(conf).equals
- (AuthenticationMethod.KERBEROS)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for OM user. "
+ "Principal: {},keytab: {}", conf.get(
OZONE_OM_KERBEROS_PRINCIPAL_KEY),
@@ -280,8 +377,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
- throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
- (conf) + " authentication method not supported. OM user login "
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+ conf) + " authentication method not supported. OM user login "
+ "failed.");
}
LOG.info("Ozone Manager login successful.");
@@ -354,7 +451,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.setPort(addr.getPort())
.setNumHandlers(handlerCount)
.setVerbose(false)
- .setSecretManager(null)
+ .setSecretManager(secretManager)
.build();
DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
@@ -394,6 +491,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
+ private static boolean isOzoneSecurityEnabled() {
+ return securityEnabled;
+ }
+
/**
* Constructs OM instance based on command line arguments.
*
@@ -438,8 +539,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
terminate(1);
return null;
}
+
+ securityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
// Authenticate KSM if security is enabled
- if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+ if (securityEnabled) {
loginOMUser(conf);
}
switch (startOpt) {
@@ -572,19 +675,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
-
- InetSocketAddress omNodeRpcAddr = getOmAddress(configuration);
- int handlerCount = configuration.getInt(OZONE_OM_HANDLER_COUNT_KEY,
- OZONE_OM_HANDLER_COUNT_DEFAULT);
- BlockingService omService = newReflectiveBlockingService(
- new OzoneManagerProtocolServerSideTranslatorPB(this));
- omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
- OzoneManagerProtocolPB.class, omService,
- handlerCount);
- omRpcAddress = updateRPCListenAddress(configuration,
- OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
- omRpcServer.start();
-
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
@@ -592,7 +682,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
DefaultMetricsSystem.initialize("OzoneManager");
metadataManager.start(configuration);
-
+ startSecretManagerIfNecessary();
// Set metrics and start metrics back ground thread
metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager
@@ -613,8 +703,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metricsTimer.schedule(scheduleOMMetricsWriteTask, 0, period);
keyManager.start(configuration);
-
- httpServer = new OzoneManagerHttpServer(configuration, this);
+ omRpcServer.start();
try {
httpServer.start();
} catch (Exception ex) {
@@ -636,6 +725,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
scheduleOMMetricsWriteTask = null;
omRpcServer.stop();
keyManager.stop();
+ stopSecretManager();
httpServer.stop();
metadataManager.stop();
metrics.unRegister();
@@ -657,6 +747,140 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
}
+ private void startSecretManagerIfNecessary() {
+ boolean shouldRun = shouldUseDelegationTokens() && isOzoneSecurityEnabled();
+ boolean running = secretManager.isRunning();
+ if (shouldRun && !running) {
+ startSecretManager();
+ }
+ }
+
+ private boolean shouldUseDelegationTokens() {
+ return UserGroupInformation.isSecurityEnabled();
+ }
+
+
+ /**
+ *
+ * @return true if delegation token operation is allowed
+ */
+ private boolean isAllowedDelegationTokenOp() throws IOException {
+ AuthenticationMethod authMethod = getConnectionAuthenticationMethod();
+ if (UserGroupInformation.isSecurityEnabled()
+ && (authMethod != AuthenticationMethod.KERBEROS)
+ && (authMethod != AuthenticationMethod.KERBEROS_SSL)
+ && (authMethod != AuthenticationMethod.CERTIFICATE)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns authentication method used to establish the connection.
+ * @return AuthenticationMethod used to establish connection
+ * @throws IOException
+ */
+ private AuthenticationMethod getConnectionAuthenticationMethod()
+ throws IOException {
+ UserGroupInformation ugi = getRemoteUser();
+ AuthenticationMethod authMethod = ugi.getAuthenticationMethod();
+ if (authMethod == AuthenticationMethod.PROXY) {
+ authMethod = ugi.getRealUser().getAuthenticationMethod();
+ }
+ return authMethod;
+ }
+
+ // optimize ugi lookup for RPC operations to avoid a trip through
+ // UGI.getCurrentUser which is synch'ed
+ private static UserGroupInformation getRemoteUser() throws IOException {
+ UserGroupInformation ugi = Server.getRemoteUser();
+ return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
+ }
+
+ /**
+ * Get delegation token from OzoneManager.
+ * @param renewer Renewer information
+ * @return delegationToken DelegationToken signed by OzoneManager
+ * @throws IOException on error
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ final boolean success;
+ final String tokenId;
+ Token<OzoneTokenIdentifier> token;
+
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException("Delegation Token can be issued only with "
+ + "kerberos or web authentication");
+ }
+ if (secretManager == null || !secretManager.isRunning()) {
+ LOG.warn("trying to get DT with no secret manager running in OM.");
+ return null;
+ }
+
+ UserGroupInformation ugi = getRemoteUser();
+ String user = ugi.getUserName();
+ Text owner = new Text(user);
+ Text realUser = null;
+ if (ugi.getRealUser() != null) {
+ realUser = new Text(ugi.getRealUser().getUserName());
+ }
+
+ token = secretManager.createToken(owner, renewer, realUser);
+ return token;
+ }
+
+ /**
+ * Method to renew a delegationToken issued by OzoneManager.
+ * @param token token to renew
+ * @return new expiryTime of the token
+ * @throws InvalidToken if {@code token} is invalid
+ * @throws IOException on other errors
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws InvalidToken, IOException {
+ long expiryTime;
+
+ try {
+
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException("Delegation Token can be renewed only with "
+ + "kerberos or web authentication");
+ }
+ String renewer = getRemoteUser().getShortUserName();
+ expiryTime = secretManager.renewToken(token, renewer);
+
+ } catch (AccessControlException ace) {
+ final OzoneTokenIdentifier id = OzoneTokenIdentifier.readProtoBuf(
+ token.getIdentifier());
+ LOG.error("Delegation token renewal failed for dt: {}, cause: {}",
+ id.toString(), ace.getMessage());
+ throw ace;
+ }
+ return expiryTime;
+ }
+
+ /**
+ * Cancels a delegation token.
+ * @param token token to cancel
+ * @throws IOException on error
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ OzoneTokenIdentifier id = null;
+ try {
+ String canceller = getRemoteUser().getUserName();
+ id = secretManager.cancelToken(token, canceller);
+ LOG.trace("Delegation token renewed for dt: {}", id);
+ } catch (AccessControlException ace) {
+ LOG.error("Delegation token renewal failed for dt: {}, cause: {}", id,
+ ace.getMessage());
+ throw ace;
+ }
+ }
/**
* Creates a volume.
*
@@ -1430,4 +1654,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return name;
}
}
+
+ public static void setTestSecureOmFlag(boolean testSecureOmFlag) {
+ OzoneManager.testSecureOmFlag = testSecureOmFlag;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 64806ed..e2536d3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -119,6 +119,15 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.SetVolumePropertyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Status;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -651,4 +660,54 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
}
return resp.build();
}
+
+ @Override
+ public GetDelegationTokenResponseProto getDelegationToken(
+ RpcController controller, GetDelegationTokenRequestProto request)
+ throws ServiceException {
+ try {
+ Token<OzoneTokenIdentifier> token = impl
+ .getDelegationToken(new Text(request.getRenewer()));
+ if (token != null) {
+ return GetDelegationTokenResponseProto.newBuilder()
+ .setToken(OMPBHelper.convertToTokenProto(token)).build();
+ }
+ return GetDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public RenewDelegationTokenResponseProto renewDelegationToken(
+ RpcController controller, RenewDelegationTokenRequestProto request)
+ throws ServiceException {
+ try {
+ if(request.hasToken()) {
+ long expiryTime = impl
+ .renewDelegationToken(
+ OMPBHelper.convertToDelegationToken(request.getToken()));
+ return RenewDelegationTokenResponseProto.newBuilder()
+ .setNewExpiryTime(expiryTime).build();
+ }
+ return RenewDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public CancelDelegationTokenResponseProto cancelDelegationToken(
+ RpcController controller, CancelDelegationTokenRequestProto req)
+ throws ServiceException {
+ try {
+ if(req.hasToken()) {
+ impl.cancelDelegationToken(
+ OMPBHelper.convertToDelegationToken(req.getToken()));
+ }
+ return CancelDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-751.
Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-751.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bee23d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bee23d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bee23d2
Branch: refs/heads/HDDS-4
Commit: 3bee23d24f6d90592e715be35fa6213a2723e0a6
Parents: 18c9db9
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:38:30 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bee23d2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index e355d0a..1bb9972 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -176,9 +176,11 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+ host.orElse(ScmConfigKeys
+ .OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
":" + port
- .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ .orElse(conf.getInt(ScmConfigKeys
+ .OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed
Certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..db5d430
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OzoneConfiguration configuration;
+ private HDDSKeyGenerator keyGenerator;
+ private String prefix;
+
+ @Before
+ public void init() throws IOException {
+ configuration = new OzoneConfiguration();
+ prefix = temporaryFolder.newFolder().toString();
+ configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+ keyGenerator = new HDDSKeyGenerator(configuration);
+ }
+
+ /**
+ * Assert basic things like we are able to create a file, and the names are
+ * in expected format etc.
+ *
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ * @throws IOException - On I/O failure.
+ */
+ @Test
+ public void testWriteKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, InvalidKeySpecException {
+ KeyPair keys = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.writeKey(keys);
+
+ // Assert that locations have been created.
+ Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+ Assert.assertTrue(keyLocation.toFile().exists());
+
+ // Assert that locations are created in the locations that we specified
+ // using the Config.
+ Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+ Path privateKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPrivateKeyFileName());
+ Assert.assertTrue(privateKeyPath.toFile().exists());
+ Path publicKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPublicKeyFileName());
+ Assert.assertTrue(publicKeyPath.toFile().exists());
+
+ // Read the private key and test if the expected String in the PEM file
+ // format exists.
+ byte[] privateKey = Files.readAllBytes(privateKeyPath);
+ String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+ // Read the public key and test if the expected String in the PEM file
+ // format exists.
+ byte[] publicKey = Files.readAllBytes(publicKeyPath);
+ String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+ // Let us decode the PEM file and parse it back into binary.
+ KeyFactory kf = KeyFactory.getInstance(
+ pemWriter.getSecurityConfig().getKeyAlgo());
+
+ // Replace the PEM Human readable guards.
+ privateKeydata =
+ privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+ privateKeydata =
+ privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+ // Decode the bas64 to binary format and then use an ASN.1 parser to
+ // parse the binary format.
+
+ byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+ PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+ PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+ Assert.assertNotNull("Private Key should not be null",
+ privateKeyDecoded);
+
+ // Let us decode the public key and veriy that we can parse it back into
+ // binary.
+ publicKeydata =
+ publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+ publicKeydata =
+ publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+ keyBytes = Base64.decodeBase64(publicKeydata);
+ X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+ PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+ Assert.assertNotNull("Public Key should not be null",
+ publicKeyDecoded);
+
+ // Now let us assert the permissions on the Directories and files are as
+ // expected.
+ Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+ Set<PosixFilePermission> currentSet =
+ Files.getPosixFilePermissions(privateKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(publicKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(keyLocation);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+ }
+
+ /**
+ * Assert key rewrite fails without force option.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testReWriteKey()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ SecurityConfig secConfig = pemWriter.getSecurityConfig();
+ pemWriter.writeKey(kp);
+
+ // Assert that rewriting of keys throws exception with valid messages.
+ LambdaTestUtils
+ .intercept(IOException.class, "Private Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPrivateKeyFileName()).toFile());
+ LambdaTestUtils
+ .intercept(IOException.class, "Public Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPublicKeyFileName()).toFile());
+
+ // Should succeed now as both public and private key are deleted.
+ pemWriter.writeKey(kp);
+ // Should succeed with overwrite flag as true.
+ pemWriter.writeKey(kp, true);
+
+ }
+
+ /**
+ * Assert key rewrite fails in non Posix file system.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testWriteKeyInNonPosixFS()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.setIsPosixFileSystem(() -> false);
+
+ // Assert key rewrite fails in non Posix file system.
+ LambdaTestUtils
+ .intercept(IOException.class, "Unsupported File System for pem file.",
+ () -> pemWriter.writeKey(kp));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..49e40b4
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test package for keys used in X.509 env.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..f541468
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * X.509 Certificate and keys related tests.
+ */
+package org.apache.hadoop.hdds.security.x509;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDFS-14095. EC: Track Erasure
Coding commands in DFS statistics. Contributed by Ayush Saxena.
Posted by xy...@apache.org.
HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5347368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5347368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5347368
Branch: refs/heads/HDDS-4
Commit: f534736867eed962899615ca1b7eb68bcf591d17
Parents: d0edd37
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Nov 30 00:18:27 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Nov 30 00:18:27 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +++
.../hadoop/hdfs/DistributedFileSystem.java | 18 ++++++
.../hadoop/hdfs/TestDistributedFileSystem.java | 63 +++++++++++++++++++-
3 files changed, 89 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index 3dcf13b..b9852ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting distributed file system operations. */
public enum OpType {
+ ADD_EC_POLICY("op_add_ec_policy"),
ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("op_concat"),
@@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics {
CREATE_SYM_LINK("op_create_symlink"),
DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("op_delete_snapshot"),
+ DISABLE_EC_POLICY("op_disable_ec_policy"),
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+ ENABLE_EC_POLICY("op_enable_ec_policy"),
EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+ GET_EC_CODECS("op_get_ec_codecs"),
+ GET_EC_POLICY("op_get_ec_policy"),
+ GET_EC_POLICIES("op_get_ec_policies"),
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("op_get_file_link_status"),
@@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics {
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+ REMOVE_EC_POLICY("op_remove_ec_policy"),
REMOVE_XATTR("op_remove_xattr"),
RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("op_resolve_link"),
SET_ACL(CommonStatisticNames.OP_SET_ACL),
+ SET_EC_POLICY("op_set_ec_policy"),
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("op_set_replication"),
@@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+ UNSET_EC_POLICY("op_unset_ec_policy"),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
private static final Map<String, OpType> SYMBOL_MAP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index ca1546c..7dd02bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2845,6 +2845,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void setErasureCodingPolicy(final Path path,
final String ecPolicyName) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
@@ -2912,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY);
Path absF = fixRelativePart(path);
return new FileSystemLinkResolver<ErasureCodingPolicy>() {
@Override
@@ -2943,6 +2947,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES);
return Arrays.asList(dfs.getErasureCodingPolicies());
}
@@ -2955,6 +2961,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Map<String, String> getAllErasureCodingCodecs()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS);
return dfs.getErasureCodingCodecs();
}
@@ -2971,6 +2979,8 @@ public class DistributedFileSystem extends FileSystem
*/
public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY);
return dfs.addErasureCodingPolicies(policies);
}
@@ -2982,6 +2992,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY);
dfs.removeErasureCodingPolicy(ecPolicyName);
}
@@ -2993,6 +3005,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void enableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY);
dfs.enableErasureCodingPolicy(ecPolicyName);
}
@@ -3004,6 +3018,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void disableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY);
dfs.disableErasureCodingPolicy(ecPolicyName);
}
@@ -3014,6 +3030,8 @@ public class DistributedFileSystem extends FileSystem
* @throws IOException
*/
public void unsetErasureCodingPolicy(final Path path) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index cae0fbf..97ced7a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -773,7 +773,68 @@ public class TestDistributedFileSystem {
} finally {
if (cluster != null) cluster.shutdown();
}
-
+ }
+
+ @Test
+ public void testECStatistics() throws IOException {
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(getTestConfiguration()).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ Path dir = new Path("/test");
+ dfs.mkdirs(dir);
+ int readOps = 0;
+ int writeOps = 0;
+ FileSystem.clearStatistics();
+
+ long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY);
+ dfs.enableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.SET_EC_POLICY);
+ dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICY);
+ dfs.getErasureCodingPolicy(dir);
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.UNSET_EC_POLICY);
+ dfs.unsetErasureCodingPolicy(dir);
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICIES);
+ dfs.getAllErasureCodingPolicies();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_CODECS);
+ dfs.getAllErasureCodingCodecs();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1);
+
+ ErasureCodingPolicy newPolicy =
+ new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024);
+
+ opCount = getOpStatistics(OpType.ADD_EC_POLICY);
+ dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy});
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.REMOVE_EC_POLICY);
+ dfs.removeErasureCodingPolicy("RS-5-3-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.DISABLE_EC_POLICY);
+ dfs.disableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1);
+ }
}
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol
interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-9. Add GRPC protocol interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96bd574d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96bd574d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96bd574d
Branch: refs/heads/HDDS-4
Commit: 96bd574dc9104ce017c889dea93792ac00900f01
Parents: 59767be
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 20 20:21:08 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../hdds/scm/ClientCredentialInterceptor.java | 65 ++++
.../hadoop/hdds/scm/XceiverClientGrpc.java | 64 +++-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +-
.../exception/SCMSecurityException.java | 52 +++
.../hdds/security/exception/package-info.java | 23 ++
.../security/token/BlockTokenException.java | 53 ++++
.../hdds/security/token/BlockTokenVerifier.java | 113 +++++++
.../token/OzoneBlockTokenIdentifier.java | 199 ++++++++++++
.../security/token/OzoneBlockTokenSelector.java | 55 ++++
.../hdds/security/token/TokenVerifier.java | 38 +++
.../hdds/security/token/package-info.java | 22 ++
.../hdds/security/x509/SecurityConfig.java | 11 +
.../authority/CertificateServer.java | 2 +-
.../certificate/client/CertificateClient.java | 11 +
.../certificates/CertificateSignRequest.java | 2 +-
.../certificates/SelfSignedCertificate.java | 2 +-
.../x509/exceptions/CertificateException.java | 14 +-
.../x509/exceptions/SCMSecurityException.java | 64 ----
.../org/apache/hadoop/ozone/OzoneConsts.java | 14 +
hadoop-hdds/common/src/main/proto/hdds.proto | 2 +-
.../token/TestOzoneBlockTokenIdentifier.java | 313 +++++++++++++++++++
.../hdds/security/token/package-info.java | 22 ++
.../TestCertificateSignRequest.java | 2 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../server/ServerCredentialInterceptor.java | 74 +++++
.../transport/server/XceiverServerGrpc.java | 23 +-
.../security/OzoneBlockTokenIdentifier.java | 178 -----------
.../ozone/security/OzoneBlockTokenSelector.java | 55 ----
.../ozoneimpl/TestSecureOzoneContainer.java | 209 +++++++++++++
.../security/TestOzoneBlockTokenIdentifier.java | 255 ---------------
30 files changed, 1363 insertions(+), 581 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
new file mode 100644
index 0000000..7a15808
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.ratis.thirdparty.io.grpc.CallOptions;
+import org.apache.ratis.thirdparty.io.grpc.Channel;
+import org.apache.ratis.thirdparty.io.grpc.ClientCall;
+import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+
+/**
+ * GRPC client interceptor for ozone block token.
+ */
+public class ClientCredentialInterceptor implements ClientInterceptor {
+
+ private final String user;
+ private final String token;
+
+ public ClientCredentialInterceptor(String user, String token) {
+ this.user = user;
+ this.token = token;
+ }
+
+ @Override
+ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
+ MethodDescriptor<ReqT, RespT> method,
+ CallOptions callOptions,
+ Channel next) {
+
+ return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
+ next.newCall(method, callOptions)) {
+ @Override
+ public void start(Listener<RespT> responseListener, Metadata headers) {
+ if (token != null) {
+ headers.put(OBT_METADATA_KEY, token);
+ }
+ if (user != null) {
+ headers.put(USER_METADATA_KEY, user);
+ }
+ super.start(responseListener, headers);
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index a824c29..ccc9372 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -31,22 +31,32 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServi
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSelector;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.Status;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import java.util.UUID;
-import java.util.Map;
-import java.util.HashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -93,7 +103,9 @@ public class XceiverClientGrpc extends XceiverClientSpi {
connectToDatanode(dn);
}
- private void connectToDatanode(DatanodeDetails dn) {
+
+ private void connectToDatanode(DatanodeDetails dn) throws IOException,
+ SCMSecurityException {
// read port from the data node, on failure use default configured
// port.
int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
@@ -101,16 +113,49 @@ public class XceiverClientGrpc extends XceiverClientSpi {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}
+
+ // Add credential context to the client call
+ String userName = UserGroupInformation.getCurrentUser()
+ .getShortUserName();
+
+ // Add block token if block token (mutual auth) is required but the client
+ // does not have a mTLS (private key and ca signed certificate)
+ String encodedToken = null;
+ SecurityConfig secConfig = new SecurityConfig(config);
+ if (secConfig.isGrpcBlockTokenEnabled()) {
+ InetSocketAddress addr = new InetSocketAddress(dn.getIpAddress(), port);
+ encodedToken = getEncodedBlockToken(addr);
+ if (encodedToken == null) {
+ throw new SCMSecurityException("No Block token available to access " +
+ "service at : " + addr.toString());
+ }
+ }
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
- ManagedChannel channel =
- NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
+ NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn
+ .getIpAddress(), port).usePlaintext()
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .build();
+ .intercept(new ClientCredentialInterceptor(userName, encodedToken));
+ ManagedChannel channel = channelBuilder.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
asyncStubs.put(dn.getUuid(), asyncStub);
channels.put(dn.getUuid(), channel);
}
+
+ private String getEncodedBlockToken(InetSocketAddress addr)
+ throws IOException{
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ OzoneBlockTokenSelector tokenSelector = new OzoneBlockTokenSelector();
+ Text service = SecurityUtil.buildTokenService(addr);
+ Token<OzoneBlockTokenIdentifier> token = tokenSelector.selectToken(
+ service, ugi.getTokens());
+ if (token != null) {
+ token.setService(service);
+ return token.encodeToUrlString();
+ }
+ return null;
+ }
+
/**
* Returns if the xceiver client connects to all servers in the pipeline.
*
@@ -173,6 +218,11 @@ public class XceiverClientGrpc extends XceiverClientSpi {
} catch (ExecutionException | InterruptedException e) {
LOG.warn("Failed to execute command " + request + " on datanode " + dn
.getUuidString(), e);
+ if (Status.fromThrowable(e.getCause()).getCode()
+ == Status.UNAUTHENTICATED.getCode()) {
+ throw new SCMSecurityException("Failed to authenticate with " +
+ "GRPC XceiverServer with Ozone block token.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index ef1194b..553f4aa 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -145,6 +145,7 @@ public final class HddsConfigKeys {
"hdds.x509.signature.algorithm";
public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
-
-
+ public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED = "hdds.grpc.block" +
+ ".token.enabled";
+ public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
new file mode 100644
index 0000000..21bc45b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.exception;
+
+import java.io.IOException;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends IOException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public SCMSecurityException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public SCMSecurityException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public SCMSecurityException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
new file mode 100644
index 0000000..b980592
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by SCM security classes.
+ */
+package org.apache.hadoop.hdds.security.exception;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
new file mode 100644
index 0000000..7ea0ebc
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
+/**
+ * Block Token Exceptions from the SCM Security layer.
+ */
+public class BlockTokenException extends SCMSecurityException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public BlockTokenException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public BlockTokenException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public BlockTokenException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
new file mode 100644
index 0000000..8c0d19e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+
+
+/**
+ * Verify token and return a UGI with token if authenticated.
+ */
+public class BlockTokenVerifier implements TokenVerifier {
+
+ private final CertificateClient caClient;
+ private final SecurityConfig conf;
+
+ public BlockTokenVerifier(SecurityConfig conf, CertificateClient caClient) {
+ this.conf = conf;
+ this.caClient = caClient;
+ }
+
+ private boolean isExpired(long expiryDate) {
+ return Time.now() > expiryDate;
+ }
+
+ @Override
+ public UserGroupInformation verify(String user, String tokenStr)
+ throws SCMSecurityException {
+ if (conf.isGrpcBlockTokenEnabled()) {
+ if (Strings.isNullOrEmpty(tokenStr)) {
+ throw new BlockTokenException("Fail to find any token (empty or " +
+ "null.");
+ }
+ final Token<OzoneBlockTokenIdentifier> token = new Token();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier();
+ try {
+ token.decodeFromUrlString(tokenStr);
+ ByteArrayInputStream buf = new ByteArrayInputStream(
+ token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ tokenId.readFields(in);
+ } catch (IOException ex) {
+ throw new BlockTokenException("Failed to decode token : " + tokenStr);
+ }
+
+ // TODO: revisit this when caClient is ready, skip signature check now.
+ /**
+ * the final code should like
+ * if (caClient == null) {
+ * throw new SCMSecurityException("Certificate client not available to
+ * validate token");
+ * }
+ */
+ if (caClient != null) {
+ X509Certificate singerCert = caClient.queryCertificate(
+ "certId=" + tokenId.getOmCertSerialId());
+ if (singerCert == null) {
+ throw new BlockTokenException("Can't find signer certificate " +
+ "(OmCertSerialId: " + tokenId.getOmCertSerialId() +
+ ") of the block token for user: " + tokenId.getUser());
+ }
+ Boolean validToken = caClient.verifySignature(tokenId.getBytes(),
+ token.getPassword(), singerCert);
+ if (!validToken) {
+ throw new BlockTokenException("Invalid block token for user: " +
+ tokenId.getUser());
+ }
+ }
+ // check expiration
+ if (isExpired(tokenId.getExpiryDate())) {
+ UserGroupInformation tokenUser = tokenId.getUser();
+ tokenUser.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.TOKEN);
+ throw new BlockTokenException("Expired block token for user: " +
+ tokenUser);
+ }
+ // defer access mode, bcsid and maxLength check to container dispatcher
+ UserGroupInformation ugi = tokenId.getUser();
+ ugi.addToken(token);
+ ugi.setAuthenticationMethod(UserGroupInformation
+ .AuthenticationMethod.TOKEN);
+ return ugi;
+ } else {
+ return UserGroupInformation.createRemoteUser(user);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..89457fd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+ static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+ private long expiryDate;
+ private String ownerId;
+ private String blockId;
+ private EnumSet<AccessModeProto> modes;
+ private String omCertSerialId;
+ private long maxLength;
+
+ public OzoneBlockTokenIdentifier() {
+ }
+
+ public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+ EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId,
+ long maxLength) {
+ this.ownerId = ownerId;
+ this.blockId = blockId;
+ this.expiryDate = expiryDate;
+ this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+ this.omCertSerialId = omCertSerialId;
+ this.maxLength = maxLength;
+ }
+
+ @Override
+ public UserGroupInformation getUser() {
+ if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+ return UserGroupInformation.createRemoteUser(blockId);
+ }
+ return UserGroupInformation.createRemoteUser(ownerId);
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public String getOwnerId() {
+ return ownerId;
+ }
+
+ public String getBlockId() {
+ return blockId;
+ }
+
+ public EnumSet<AccessModeProto> getAccessModes() {
+ return modes;
+ }
+
+ public String getOmCertSerialId(){
+ return omCertSerialId;
+ }
+
+ public long getMaxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String toString() {
+ return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ + ", ownerId=" + this.getOwnerId()
+ + ", omCertSerialId=" + this.getOmCertSerialId()
+ + ", blockId=" + this.getBlockId() + ", access modes="
+ + this.getAccessModes() + ", maxLength=" + this.getMaxLength() + ")";
+ }
+
+ static boolean isEqual(Object a, Object b) {
+ return a == null ? b == null : a.equals(b);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneBlockTokenIdentifier) {
+ OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+ return new EqualsBuilder()
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.ownerId, that.ownerId)
+ .append(this.blockId, that.blockId)
+ .append(this.modes, that.modes)
+ .append(this.omCertSerialId, that.omCertSerialId)
+ .append(this.maxLength, that.maxLength)
+ .build();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(133, 567)
+ .append(this.expiryDate)
+ .append(this.blockId)
+ .append(this.ownerId)
+ .append(this.modes)
+ .append(this.omCertSerialId)
+ .append(this.maxLength)
+ .build();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final DataInputStream dis = (DataInputStream) in;
+ if (!dis.markSupported()) {
+ throw new IOException("Could not peek first byte.");
+ }
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ this.ownerId = tokenPtoto.getOwnerId();
+ this.blockId = tokenPtoto.getBlockId();
+ this.modes = EnumSet.copyOf(tokenPtoto.getModesList());
+ this.expiryDate = tokenPtoto.getExpiryDate();
+ this.omCertSerialId = tokenPtoto.getOmCertSerialId();
+ this.maxLength = tokenPtoto.getMaxLength();
+ }
+
+ @VisibleForTesting
+ public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+ throws IOException {
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+ tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+ tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId(),
+ tokenPtoto.getMaxLength());
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ writeProtobuf(out);
+ }
+
+ @VisibleForTesting
+ void writeProtobuf(DataOutput out) throws IOException {
+ Builder builder = BlockTokenSecretProto.newBuilder()
+ .setBlockId(this.getBlockId())
+ .setOwnerId(this.getOwnerId())
+ .setOmCertSerialId(this.getOmCertSerialId())
+ .setExpiryDate(this.getExpiryDate())
+ .setMaxLength(this.getMaxLength());
+ // Add access mode allowed
+ for (AccessModeProto mode : this.getAccessModes()) {
+ builder.addModes(AccessModeProto.valueOf(mode.name()));
+ }
+ out.write(builder.build().toByteArray());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..1943cce
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+ TokenSelector<OzoneBlockTokenIdentifier> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneBlockTokenSelector.class);
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+ LOG.trace("Getting token for service:{}", service);
+ return (Token<OzoneBlockTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
new file mode 100644
index 0000000..d8170ab
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Ozone GRPC token header verifier.
+ */
+public interface TokenVerifier {
+ /**
+ * Given a user and tokenStr header, return a UGI object with token if
+ * verified.
+ * @param user user of the request
+ * @param tokenStr token str of the request
+ * @return UGI
+ * @throws SCMSecurityException
+ */
+ UserGroupInformation verify(String user, String tokenStr)
+ throws SCMSecurityException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..885bed5
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related test classes.
+ */
+package org.apache.hadoop.hdds.security.token;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 9231c8a..97627ca 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -34,6 +34,8 @@ import java.time.Duration;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
@@ -70,6 +72,7 @@ public class SecurityConfig {
private final String publicKeyFileName;
private final Duration certDuration;
private final String x509SignatureAlgo;
+ private final Boolean grpcBlockTokenEnabled;
/**
* Constructs a SecurityConfig.
@@ -106,6 +109,10 @@ public class SecurityConfig {
this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+ this.grpcBlockTokenEnabled = this.configuration.getBoolean(
+ HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+ HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT);
+
// First Startup -- if the provider is null, check for the provider.
if (SecurityConfig.provider == null) {
synchronized (SecurityConfig.class) {
@@ -213,6 +220,10 @@ public class SecurityConfig {
return this.certDuration;
}
+ public Boolean isGrpcBlockTokenEnabled() {
+ return this.grpcBlockTokenEnabled;
+ }
+
/**
* Adds a security provider dynamically if it is not loaded already.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index 9332e5b..5b5deef 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hdds.security.x509.certificate.authority;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.bouncycastle.cert.X509CertificateHolder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index 1b6f576..bfc0576 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -91,6 +91,17 @@ public interface CertificateClient {
X509Certificate cert);
/**
+ * Verifies a digital Signature, given the signature and the certificate of
+ * the signer.
+ * @param data - Data in byte array.
+ * @param signature - Byte Array containing the signature.
+ * @param cert - Certificate of the Signer.
+ * @return true if verified, false if not.
+ */
+ boolean verifySignature(byte[] data, byte[] signature,
+ X509Certificate cert);
+
+ /**
* Returns a CSR builder that can be used to creates a Certificate sigining
* request.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
index 2e1f9df..0e762a5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.DEROctetString;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index f221246..85fba9b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -22,9 +22,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.util.Time;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.x500.X500Name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
index 4d2dcfd..5a8bf86 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hdds.security.x509.exceptions;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
/**
* Certificate Exceptions from the SCM Security layer.
*/
@@ -48,16 +50,4 @@ public class CertificateException extends SCMSecurityException {
public CertificateException(Throwable cause) {
super(cause);
}
-
- /**
- * Ctor.
- * @param message - Error Message
- * @param cause - Cause
- * @param enableSuppression - Enable suppression.
- * @param writableStackTrace - Writable stack trace.
- */
- public CertificateException(String message, Throwable cause,
- boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
deleted file mode 100644
index e1f6296..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.exceptions;
-
-/**
- * Root Security Exception call for all Certificate related Execptions.
- */
-public class SCMSecurityException extends Exception {
-
- /**
- * Ctor.
- * @param message - Error Message.
- */
- public SCMSecurityException(String message) {
- super(message);
- }
-
- /**
- * Ctor.
- * @param message - Message.
- * @param cause - Actual cause.
- */
- public SCMSecurityException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * Ctor.
- * @param cause - Base Exception.
- */
- public SCMSecurityException(Throwable cause) {
- super(cause);
- }
-
-
- /**
- * Ctor.
- * @param message - Error Message
- * @param cause - Cause
- * @param enableSuppression - Enable suppression.
- * @param writableStackTrace - Writable stack trace.
- */
- public SCMSecurityException(String message, Throwable cause,
- boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index e7c354c..dad3bfe 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -19,6 +19,11 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+
+import static org.apache.ratis.thirdparty.io.grpc.Metadata.ASCII_STRING_MARSHALLER;
/**
* Set of constants used in Ozone implementation.
@@ -231,4 +236,13 @@ public final class OzoneConsts {
public static final String OM_METRICS_FILE = "omMetrics";
public static final String OM_METRICS_TEMP_FILE = OM_METRICS_FILE + ".tmp";
+ // GRPC block token metadata header and context key
+ public static final String OZONE_BLOCK_TOKEN = "blocktoken";
+ public static final Context.Key<UserGroupInformation> UGI_CTX_KEY =
+ Context.key("UGI");
+
+ public static final Metadata.Key<String> OBT_METADATA_KEY =
+ Metadata.Key.of(OZONE_BLOCK_TOKEN, ASCII_STRING_MARSHALLER);
+ public static final Metadata.Key<String> USER_METADATA_KEY =
+ Metadata.Key.of(OZONE_USER, ASCII_STRING_MARSHALLER);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 8598fbf..53d11a5 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -210,7 +210,7 @@ message BlockTokenSecretProto {
required uint64 expiryDate = 3;
required string omCertSerialId = 4;
repeated AccessModeProto modes = 5;
-
+ required uint64 maxLength = 6;
}
message BlockID {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..77a2cec
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneBlockTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static long expiryTime;
+ private static KeyPair keyPair;
+ private static X509Certificate cert;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 128L);
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneBlockTokenIdentifier("", "",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 128L);
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ @Test
+ public void testTokenSerialization() throws GeneralSecurityException,
+ IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+ long maxLength = 128L;
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), maxLength);
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+
+ Token<BlockTokenIdentifier> token = new Token(tokenId.getBytes(),
+ signedToken, tokenId.getKind(), new Text("host:port"));
+
+ String encodeToUrlString = token.encodeToUrlString();
+
+ Token<BlockTokenIdentifier>decodedToken = new Token();
+ decodedToken.decodeFromUrlString(encodeToUrlString);
+
+ OzoneBlockTokenIdentifier decodedTokenId = new OzoneBlockTokenIdentifier();
+ decodedTokenId.readFields(new DataInputStream(
+ new ByteArrayInputStream(decodedToken.getIdentifier())));
+
+ Assert.assertEquals(decodedTokenId, tokenId);
+ Assert.assertEquals(decodedTokenId.getMaxLength(), maxLength);
+
+ // Verify a decoded signed Token with public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(decodedTokenId, decodedToken
+ .getPassword(), cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+ }
+
+
+ public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValid = rsaSignature.verify(signature);
+ return isValid;
+ }
+
+ private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneBlockTokenIdentifier generateTestToken() {
+ return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+ RandomStringUtils.randomAlphabetic(5),
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 1024768L);
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate certificate;
+ certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ kp, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration / testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+ certificate);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration / testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration / testTokenCount);
+ }
+
+ // TODO: verify certificate with a trust store
+ public boolean verifyCert(Certificate certificate) {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..d056655
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related classes.
+ */
+package org.apache.hadoop.hdds.security.token;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 0b9ef31..e8de1ea 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.bouncycastle.asn1.ASN1Sequence;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 3a00ca0..16245a6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -20,8 +20,8 @@
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.cert.X509CertificateHolder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
new file mode 100644
index 0000000..968f0c8
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.transport.server;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.TokenVerifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Contexts;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.ServerCall;
+import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.Status;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.UGI_CTX_KEY;
+/**
+ * Grpc Server Interceptor for Ozone Block token.
+ */
+public class ServerCredentialInterceptor implements ServerInterceptor {
+
+
+ private static final ServerCall.Listener NOOP_LISTENER =
+ new ServerCall.Listener() {
+ };
+
+ private final TokenVerifier verifier;
+
+ ServerCredentialInterceptor(TokenVerifier verifier) {
+ this.verifier = verifier;
+ }
+
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
+ ServerCall<ReqT, RespT> call, Metadata headers,
+ ServerCallHandler<ReqT, RespT> next) {
+ String token = headers.get(OBT_METADATA_KEY);
+ String user = headers.get(USER_METADATA_KEY);
+ Context ctx = Context.current();
+ try {
+ UserGroupInformation ugi = verifier.verify(user, token);
+ if (ugi == null) {
+ call.close(Status.UNAUTHENTICATED.withDescription("Missing Block " +
+ "Token from headers when block token is required."), headers);
+ return NOOP_LISTENER;
+ } else {
+ ctx = ctx.withValue(UGI_CTX_KEY, ugi);
+ }
+ } catch (SCMSecurityException e) {
+ call.close(Status.UNAUTHENTICATED.withDescription(e.getMessage())
+ .withCause(e), headers);
+ return NOOP_LISTENER;
+ }
+ return Contexts.interceptCall(ctx, call, headers, next);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4f7799d..2a3c99b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.transport.server;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -30,6 +31,9 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
+import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -37,6 +41,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
import org.apache.ratis.thirdparty.io.grpc.Server;
import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,8 +97,16 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .addService(new GrpcXceiverService(dispatcher));
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+
+ // Populate UGI context via ServerCredentialInterceptor
+ SecurityConfig secConfig = new SecurityConfig(conf);
+ ServerCredentialInterceptor credInterceptor =
+ new ServerCredentialInterceptor(
+ new BlockTokenVerifier(secConfig, getCaClient()));
+ nettyServerBuilder.addService(ServerInterceptors.intercept(
+ new GrpcXceiverService(dispatcher), credInterceptor));
+
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
}
@@ -101,6 +114,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
storageContainer = dispatcher;
}
+ @VisibleForTesting
+ public CertificateClient getCaClient() {
+ // TODO: instantiate CertificateClient
+ return null;
+ }
+
@Override
public int getIPCPort() {
return this.port;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
deleted file mode 100644
index 92b6d16..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.security;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.TokenIdentifier;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.EnumSet;
-
-/**
- * Block token identifier for Ozone/HDDS. Ozone block access token is similar
- * to HDFS block access token, which is meant to be lightweight and
- * short-lived. No need to renew or revoke a block access token. when a
- * cached block access token expires, the client simply get a new one.
- * Block access token should be cached only in memory and never write to disk.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenIdentifier extends TokenIdentifier {
-
- static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
- private long expiryDate;
- private String ownerId;
- private String blockId;
- private final EnumSet<AccessModeProto> modes;
- private final String omCertSerialId;
-
- public OzoneBlockTokenIdentifier(String ownerId, String blockId,
- EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
- this.ownerId = ownerId;
- this.blockId = blockId;
- this.expiryDate = expiryDate;
- this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
- this.omCertSerialId = omCertSerialId;
- }
-
- @Override
- public UserGroupInformation getUser() {
- if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
- return UserGroupInformation.createRemoteUser(blockId);
- }
- return UserGroupInformation.createRemoteUser(ownerId);
- }
-
- public long getExpiryDate() {
- return expiryDate;
- }
-
- public String getOwnerId() {
- return ownerId;
- }
-
- public String getBlockId() {
- return blockId;
- }
-
- public EnumSet<AccessModeProto> getAccessModes() {
- return modes;
- }
-
- public String getOmCertSerialId(){
- return omCertSerialId;
- }
-
- @Override
- public Text getKind() {
- return KIND_NAME;
- }
-
- @Override
- public String toString() {
- return "block_token_identifier (expiryDate=" + this.getExpiryDate()
- + ", ownerId=" + this.getOwnerId()
- + ", omCertSerialId=" + this.getOmCertSerialId()
- + ", blockId=" + this.getBlockId() + ", access modes="
- + this.getAccessModes() + ")";
- }
-
- static boolean isEqual(Object a, Object b) {
- return a == null ? b == null : a.equals(b);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
-
- if (obj instanceof OzoneBlockTokenIdentifier) {
- OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
- return new EqualsBuilder()
- .append(this.expiryDate, that.expiryDate)
- .append(this.ownerId, that.ownerId)
- .append(this.blockId, that.blockId)
- .append(this.modes, that.modes)
- .append(this.omCertSerialId, that.omCertSerialId)
- .build();
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(133, 567)
- .append(this.expiryDate)
- .append(this.blockId)
- .append(this.ownerId)
- .append(this.modes)
- .append(this.omCertSerialId)
- .build();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- final DataInputStream dis = (DataInputStream) in;
- if (!dis.markSupported()) {
- throw new IOException("Could not peek first byte.");
- }
- readFieldsProtobuf(dis);
- }
-
- @VisibleForTesting
- public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
- throws IOException {
- BlockTokenSecretProto tokenPtoto =
- BlockTokenSecretProto.parseFrom((DataInputStream) in);
- return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
- tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
- tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- writeProtobuf(out);
- }
-
- @VisibleForTesting
- void writeProtobuf(DataOutput out) throws IOException {
- Builder builder = BlockTokenSecretProto.newBuilder()
- .setBlockId(this.getBlockId())
- .setOwnerId(this.getOwnerId())
- .setOmCertSerialId(this.getOmCertSerialId())
- .setExpiryDate(this.getExpiryDate());
- // Add access mode allowed
- for (AccessModeProto mode : this.getAccessModes()) {
- builder.addModes(AccessModeProto.valueOf(mode.name()));
- }
- out.write(builder.build().toByteArray());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96bd574d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
deleted file mode 100644
index 086e19b..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-
-/**
- * A block token selector for Ozone.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenSelector implements
- TokenSelector<OzoneBlockTokenIdentifier> {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(OzoneBlockTokenSelector.class);
-
- @Override
- @SuppressWarnings("unchecked")
- public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
- if (service == null) {
- return null;
- }
- for (Token<? extends TokenIdentifier> token : tokens) {
- if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
- LOG.trace("Getting token for service:{}", service);
- return (Token<OzoneBlockTokenIdentifier>) token;
- }
- }
- return null;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDDS-70. Fix config names for
secure ksm and scm. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8af727f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8af727f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8af727f
Branch: refs/heads/HDDS-4
Commit: c8af727fa5efb58e9bfaeb2816d7d63c53e0bd62
Parents: 5c82193
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 22 13:32:28 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:50 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 2 +-
.../StorageContainerLocationProtocol.java | 3 +-
.../protocolPB/ScmBlockLocationProtocolPB.java | 4 +-
.../StorageContainerLocationProtocolPB.java | 2 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 1 -
.../common/src/main/resources/ozone-default.xml | 31 +++++---
.../StorageContainerDatanodeProtocol.java | 2 +-
.../StorageContainerDatanodeProtocolPB.java | 2 +-
.../scm/server/StorageContainerManager.java | 18 ++---
.../compose/compose-secure/docker-compose.yaml | 6 +-
.../test/compose/compose-secure/docker-config | 12 +--
.../acceptance/ozone-secure.robot | 12 +--
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 ++++++++++++++++++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 21 +++--
.../apache/hadoop/ozone/om/OzoneManager.java | 4 +-
18 files changed, 151 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index e18fe91..376e6db 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -192,9 +192,9 @@ public final class ScmConfigKeys {
"ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address";
- public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.scm.kerberos.keytab.file";
- public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
+ public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.kerberos.keytab.file";
+ public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index e17f1c2..2d46ae0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -33,7 +33,7 @@ import java.util.List;
* ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
* to read/write a block.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 5bc2521..e21bc53 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm.protocol;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerLocationProtocol {
/**
* Asks SCM where a container should be allocated. SCM responds with the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 89bb066..06bbd05 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,11 +18,9 @@
package org.apache.hadoop.hdds.scm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
.ScmBlockLocationProtocolService;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.security.KerberosInfo;
@@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
protocolVersion = 1)
@InterfaceAudience.Private
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocolPB
extends ScmBlockLocationProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index 3bd83f9..f80ba20 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
protocolVersion = 1)
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerLocationProtocolPB
extends StorageContainerLocationProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index b47113b..578a983 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -334,7 +334,6 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
-
/**
* Config properties to set client side checksum properties.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index d3e352b..13b47c9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -500,6 +500,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
@@ -509,6 +510,9 @@
</property>
<property>
<name>ozone.om.db.cache.size.mb</name>
+=======
+ <name>ozone.ksm.db.cache.size.mb</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>128</value>
<tag>OM, PERFORMANCE</tag>
<description>
@@ -1019,7 +1023,7 @@
</property>
<property>
- <name>ozone.scm.container.creation.lease.timeout</name>
+ <name>hdds.scm.container.creation.lease.timeout</name>
<value>60s</value>
<tag>OZONE, SCM</tag>
<description>
@@ -1073,7 +1077,11 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>hdds.container.close.threshold</name>
+=======
+ <name>hdds.scm.container.close.threshold</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>0.9f</value>
<tag>OZONE, DATANODE</tag>
<description>
@@ -1278,6 +1286,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>hdds.db.profile</name>
<value>DISK</value>
<tag>OZONE, OM, PERFORMANCE</tag>
@@ -1388,16 +1397,16 @@
</property>
<property>
- <name>ozone.scm.kerberos.keytab.file</name>
+ <name>hdds.scm.kerberos.keytab.file</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description> The keytab file used by each SCM daemon to login as its
service principal. The principal name is configured with
- ozone.scm.kerberos.principal.
+ hdds.scm.kerberos.principal.
</description>
</property>
<property>
- <name>ozone.scm.kerberos.principal</name>
+ <name>hdds.scm.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
@@ -1407,24 +1416,24 @@
<name>ozone.om.kerberos.keytab.file</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description> The keytab file used by KSM daemon to login as its
+ <description> The keytab file used by OzoneManager daemon to login as its
service principal. The principal name is configured with
- hdds.ksm.kerberos.principal.
+ ozone.om.kerberos.principal.
</description>
</property>
<property>
<name>ozone.om.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
+ <description>The OzoneManager service principal. Ex om/_HOST@REALM.COM</description>
</property>
<property>
- <name>ozone.scm.web.authentication.kerberos.principal</name>
+ <name>hdds.scm.web.authentication.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
</property>
<property>
- <name>ozone.scm.web.authentication.kerberos.keytab</name>
+ <name>hdds.scm.web.authentication.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
@@ -1432,14 +1441,14 @@
<name>ozone.om.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<description>
- KSM http server kerberos principal.
+ OzoneManager http server kerberos principal.
</description>
</property>
<property>
<name>ozone.om.http.kerberos.keytab.file</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
- KSM http server kerberos keytab.
+ OzoneManager http server kerberos keytab.
</description>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index 8049e9d..3600581 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo;
* Protoc file that defines this protocol.
*/
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerDatanodeProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9c32ef8..9006e91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
protocolVersion = 1)
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerDatanodeProtocolPB extends
StorageContainerDatanodeProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 519f51b..27fe326 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -108,8 +108,8 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -332,17 +332,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private void loginAsSCMUser(Configuration conf)
throws IOException, AuthenticationException {
LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
- + "Principal: {}, keytab: {}", conf.get
- (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
- conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+ + "Principal: {}, keytab: {}",
+ conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY));
- if (SecurityUtil.getAuthenticationMethod(conf).equals
- (AuthenticationMethod.KERBEROS)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ AuthenticationMethod.KERBEROS)) {
UserGroupInformation.setConfiguration(conf);
InetSocketAddress socAddr = HddsServerUtil
.getScmBlockClientBindAddress(conf);
- SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
- OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
conf) + " authentication method not support. "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
index 2661163..db211bc 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -40,15 +40,15 @@ services:
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","datanode"]
- ksm:
+ om:
image: ahadoop/ozone:v1
- hostname: ksm
+ hostname: om
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9874:9874
environment:
- ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 678c75a..360b69a 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.address=om
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
@@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.client.address=scm
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
@@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYTABS=dn nn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
index 4a78980..7fc1088 100644
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -26,7 +26,7 @@ ${version}
*** Test Cases ***
Daemons are running
- Is daemon running ksm
+ Is daemon running om
Is daemon running scm
Is daemon running datanode
Is daemon running ozone.kdc
@@ -45,15 +45,15 @@ Test rest interface
Should contain ${result} 200 OK
Test ozone cli
- ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
Should contain ${result} Client cannot authenticate via
# Authenticate testuser
Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
- Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
- ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+ Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
- Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
- ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+ Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
+ ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
Should Be Equal ${result} bill
*** Keywords ***
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index f5b2cd9..17634f2 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.security.KerberosInfo;
* includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
* {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
new file mode 100644
index 0000000..cc25dbe
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+
+import org.apache.hadoop.ozone.OzoneAcl;
+/**
+ * KSM Constants.
+ */
+public final class KSMConfigKeys {
+ /**
+ * Never constructed.
+ */
+ private KSMConfigKeys() {
+ }
+
+
+ public static final String OZONE_KSM_HANDLER_COUNT_KEY =
+ "ozone.ksm.handler.count.key";
+ public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
+
+ public static final String OZONE_KSM_ADDRESS_KEY =
+ "ozone.ksm.address";
+ public static final String OZONE_KSM_BIND_HOST_DEFAULT =
+ "0.0.0.0";
+ public static final int OZONE_KSM_PORT_DEFAULT = 9862;
+
+ public static final String OZONE_KSM_HTTP_ENABLED_KEY =
+ "ozone.ksm.http.enabled";
+ public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
+ "ozone.ksm.http-bind-host";
+ public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
+ "ozone.ksm.https-bind-host";
+ public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
+ "ozone.ksm.http-address";
+ public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
+ "ozone.ksm.https-address";
+ public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
+ public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
+ public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
+
+ // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
+ public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
+ "ozone.ksm.db.cache.size.mb";
+ public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
+
+ public static final String OZONE_KSM_USER_MAX_VOLUME =
+ "ozone.ksm.user.max.volume";
+ public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
+
+ // KSM Default user/group permissions
+ public static final String OZONE_KSM_USER_RIGHTS =
+ "ozone.ksm.user.rights";
+ public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
+ OzoneAcl.OzoneACLRights.READ_WRITE;
+
+ public static final String OZONE_KSM_GROUP_RIGHTS =
+ "ozone.ksm.group.rights";
+ public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
+ OzoneAcl.OzoneACLRights.READ_WRITE;
+
+ public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
+ "ozone.key.deleting.limit.per.task";
+ public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
+
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
+ "ozone.om.web.authentication.kerberos.principal";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ "ozone.om.web.authentication.kerberos.keytab";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 2a4e864..233666a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -26,10 +26,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
-
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.security.KerberosInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5c96067..3fb5499 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -118,10 +118,10 @@ public final class TestSecureOzoneCluster {
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception {
createPrincipal(scmKeytab,
- conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
createPrincipal(omKeyTab,
conf.get(OMConfigKeys
.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
@@ -154,7 +154,7 @@ public final class TestSecureOzoneCluster {
"kerberos");
conf.set(OZONE_ADMINISTRATORS, curUser);
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
@@ -162,19 +162,18 @@ public final class TestSecureOzoneCluster {
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
- "HTTP_KSM/" + host + "@" + realm);
+ "HTTP_OM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
omKeyTab = new File(workDir, "om.keytab");
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
-
}
@Test
@@ -205,7 +204,7 @@ public final class TestSecureOzoneCluster {
@Test
public void testSecureScmStartupFailure() throws Exception {
initSCM();
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
@@ -215,9 +214,9 @@ public final class TestSecureOzoneCluster {
StorageContainerManager.createSCM(null, conf);
});
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/_HOST@EXAMPLE.com");
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab");
testCommonKerberosFailures(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8af727f/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a05b2fd..7fc6aee 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -270,8 +270,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
- + "Principal: {},keytab: {}", conf.get
- (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+ + "Principal: {},keytab: {}", conf.get(
+ OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
UserGroupInformation.setConfiguration(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-8975. [Submarine] Use
predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8.
(Zhankun Tang via wangda)
Posted by xy...@apache.org.
YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)
Change-Id: If6c7904aa17895e543cfca245264249eb7328bdc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89764392
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89764392
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89764392
Branch: refs/heads/HDDS-4
Commit: 897643928c534062d45d00a36b95fd222299b4f6
Parents: 8ebeda9
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:39:06 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:39:06 2018 -0800
----------------------------------------------------------------------
.../submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89764392/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index b58ad77..2e84c96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -49,6 +49,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -218,7 +219,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private String generateCommandLaunchScript(RunJobParameters parameters,
TaskType taskType, Component comp) throws IOException {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
- Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream(file),
+ StandardCharsets.UTF_8);
PrintWriter pw = new PrintWriter(w);
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDDS-696. Bootstrap genesis
SCM(CA) with self-signed certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 459dce7..ded50f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -28,7 +28,9 @@ import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
-/** A class to generate Key Pair for use with Certificates. */
+/**
+ * A class to generate Key Pair for use with Certificates.
+ */
public class HDDSKeyGenerator {
private static final Logger LOG =
LoggerFactory.getLogger(HDDSKeyGenerator.class);
@@ -44,7 +46,17 @@ public class HDDSKeyGenerator {
}
/**
+ * Constructor that takes a SecurityConfig as the Argument.
+ *
+ * @param config - SecurityConfig
+ */
+ public HDDSKeyGenerator(SecurityConfig config) {
+ this.securityConfig = config;
+ }
+
+ /**
* Returns the Security config used for this object.
+ *
* @return SecurityConfig
*/
public SecurityConfig getSecurityConfig() {
@@ -55,10 +67,10 @@ public class HDDSKeyGenerator {
* Use Config to generate key.
*
* @return KeyPair
- * @throws NoSuchProviderException - On Error, due to missing Java
- * dependencies.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
* @throws NoSuchAlgorithmException - On Error, due to missing Java
- * dependencies.
+ * dependencies.
*/
public KeyPair generateKey() throws NoSuchProviderException,
NoSuchAlgorithmException {
@@ -71,10 +83,10 @@ public class HDDSKeyGenerator {
*
* @param size - int, valid key sizes.
* @return KeyPair
- * @throws NoSuchProviderException - On Error, due to missing Java
- * dependencies.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
* @throws NoSuchAlgorithmException - On Error, due to missing Java
- * dependencies.
+ * dependencies.
*/
public KeyPair generateKey(int size) throws
NoSuchProviderException, NoSuchAlgorithmException {
@@ -89,10 +101,10 @@ public class HDDSKeyGenerator {
* @param algorithm - Algorithm to use
* @param provider - Security provider.
* @return KeyPair.
- * @throws NoSuchProviderException - On Error, due to missing Java
- * dependencies.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
* @throws NoSuchAlgorithmException - On Error, due to missing Java
- * dependencies.
+ * dependencies.
*/
public KeyPair generateKey(int size, String algorithm, String provider)
throws NoSuchProviderException, NoSuchAlgorithmException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
deleted file mode 100644
index 95be1c4..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509.keys;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.output.FileWriterWithEncoding;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyPair;
-import java.util.Set;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * We store all Key material in good old PEM files.
- * This helps in avoiding dealing will persistent
- * Java KeyStore issues. Also when debugging,
- * general tools like OpenSSL can be used to read and
- * decode these files.
- */
-public class HDDSKeyPEMWriter {
- private static final Logger LOG =
- LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
- private final Path location;
- private final SecurityConfig securityConfig;
- private Set<PosixFilePermission> permissionSet =
- Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
- .collect(Collectors.toSet());
- private Supplier<Boolean> isPosixFileSystem;
- public final static String PRIVATE_KEY = "PRIVATE KEY";
- public final static String PUBLIC_KEY = "PUBLIC KEY";
- public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
- /*
- Creates an HDDS Key Writer.
-
- @param configuration - Configuration
- */
- public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration, "Config cannot be null");
- this.securityConfig = new SecurityConfig(configuration);
- isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
- this.location = securityConfig.getKeyLocation();
- }
-
- /**
- * Checks if File System supports posix style security permissions.
- *
- * @return True if it supports posix.
- */
- private static Boolean isPosix() {
- return FileSystems.getDefault().supportedFileAttributeViews()
- .contains("posix");
- }
-
- /**
- * Returns the Permission set.
- * @return Set
- */
- @VisibleForTesting
- public Set<PosixFilePermission> getPermissionSet() {
- return permissionSet;
- }
-
- /**
- * Returns the Security config used for this object.
- * @return SecurityConfig
- */
- public SecurityConfig getSecurityConfig() {
- return securityConfig;
- }
-
- /**
- * This function is used only for testing.
- *
- * @param isPosixFileSystem - Sets a boolean function for mimicking
- * files systems that are not posix.
- */
- @VisibleForTesting
- public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
- this.isPosixFileSystem = isPosixFileSystem;
- }
-
- /**
- * Writes a given key using the default config options.
- *
- * @param keyPair - Key Pair to write to file.
- * @throws IOException - On I/O failure.
- */
- public void writeKey(KeyPair keyPair) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
- securityConfig.getPublicKeyFileName(), false);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException - On I/O failure.
- */
- public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
- securityConfig.getPublicKeyFileName(), overwrite);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param basePath - The location to write to, override the config values.
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException - On I/O failure.
- */
- public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
- throws IOException {
- writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
- securityConfig.getPublicKeyFileName(), overwrite);
- }
-
- /**
- * Helper function that actually writes data to the files.
- *
- * @param basePath - base path to write key
- * @param keyPair - Key pair to write to file.
- * @param privateKeyFileName - private key file name.
- * @param publicKeyFileName - public key file name.
- * @param force - forces overwriting the keys.
- * @throws IOException - On I/O failure.
- */
- private synchronized void writeKey(Path basePath, KeyPair keyPair,
- String privateKeyFileName, String publicKeyFileName, boolean force)
- throws IOException {
- checkPreconditions(basePath);
-
- File privateKeyFile =
- Paths.get(location.toString(), privateKeyFileName).toFile();
- File publicKeyFile =
- Paths.get(location.toString(), publicKeyFileName).toFile();
- checkKeyFile(privateKeyFile, force, publicKeyFile);
-
- try (PemWriter privateKeyWriter = new PemWriter(new
- FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
- privateKeyWriter.writeObject(
- new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
- }
-
- try (PemWriter publicKeyWriter = new PemWriter(new
- FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
- publicKeyWriter.writeObject(
- new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
- }
- Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
- Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
- }
-
- /**
- * Checks if private and public key file already exists. Throws IOException
- * if file exists and force flag is set to false, else will delete the
- * existing file.
- *
- * @param privateKeyFile - Private key file.
- * @param force - forces overwriting the keys.
- * @param publicKeyFile - public key file.
- * @throws IOException - On I/O failure.
- */
- private void checkKeyFile(File privateKeyFile, boolean force,
- File publicKeyFile) throws IOException {
- if (privateKeyFile.exists() && force) {
- if (!privateKeyFile.delete()) {
- throw new IOException("Unable to delete private key file.");
- }
- }
-
- if (publicKeyFile.exists() && force) {
- if (!publicKeyFile.delete()) {
- throw new IOException("Unable to delete public key file.");
- }
- }
-
- if (privateKeyFile.exists()) {
- throw new IOException("Private Key file already exists.");
- }
-
- if (publicKeyFile.exists()) {
- throw new IOException("Public Key file already exists.");
- }
- }
-
- /**
- * Checks if base path exists and sets file permissions.
- *
- * @param basePath - base path to write key
- * @throws IOException - On I/O failure.
- */
- private void checkPreconditions(Path basePath) throws IOException {
- Preconditions.checkNotNull(basePath, "Base path cannot be null");
- if (!isPosixFileSystem.get()) {
- LOG.error("Keys cannot be stored securely without POSIX file system "
- + "support for now.");
- throw new IOException("Unsupported File System for pem file.");
- }
-
- if (Files.exists(basePath)) {
- // Not the end of the world if we reset the permissions on an existing
- // directory.
- Files.setPosixFilePermissions(basePath, permissionSet);
- } else {
- boolean success = basePath.toFile().mkdirs();
- if (!success) {
- LOG.error("Unable to create the directory for the "
- + "location. Location: {}", basePath);
- throw new IOException("Unable to create the directory for the "
- + "location. Location:" + basePath);
- }
- Files.setPosixFilePermissions(basePath, permissionSet);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
new file mode 100644
index 0000000..1d45ef1
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files. This helps in avoiding
+ * dealing will persistent Java KeyStore issues. Also when debugging, general
+ * tools like OpenSSL can be used to read and decode these files.
+ */
+public class KeyCodec {
+ public final static String PRIVATE_KEY = "PRIVATE KEY";
+ public final static String PUBLIC_KEY = "PUBLIC KEY";
+ public final static Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+ private final static Logger LOG =
+ LoggerFactory.getLogger(KeyCodec.class);
+ private final Path location;
+ private final SecurityConfig securityConfig;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ private Supplier<Boolean> isPosixFileSystem;
+
+ /**
+ * Creates an KeyCodec.
+ *
+ * @param config - Security Config.
+ * @param component - Component String.
+ */
+ public KeyCodec(SecurityConfig config, String component) {
+ this.securityConfig = config;
+ isPosixFileSystem = KeyCodec::isPosix;
+ this.location = securityConfig.getKeyLocation(component);
+ }
+
+ /**
+ * Creates an HDDS Key Writer.
+ *
+ * @param configuration - Configuration
+ */
+ public KeyCodec(Configuration configuration) {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ isPosixFileSystem = KeyCodec::isPosix;
+ this.location = securityConfig.getKeyLocation();
+ }
+
+ /**
+ * Checks if File System supports posix style security permissions.
+ *
+ * @return True if it supports posix.
+ */
+ private static Boolean isPosix() {
+ return FileSystems.getDefault().supportedFileAttributeViews()
+ .contains("posix");
+ }
+
+ /**
+ * Returns the Permission set.
+ *
+ * @return Set
+ */
+ @VisibleForTesting
+ public Set<PosixFilePermission> getPermissionSet() {
+ return permissionSet;
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ *
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * This function is used only for testing.
+ *
+ * @param isPosixFileSystem - Sets a boolean function for mimicking files
+ * systems that are not posix.
+ */
+ @VisibleForTesting
+ public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+ this.isPosixFileSystem = isPosixFileSystem;
+ }
+
+ /**
+ * Writes a given key using the default config options.
+ *
+ * @param keyPair - Key Pair to write to file.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), false);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param basePath - The location to write to, override the config values.
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+ throws IOException {
+ writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Reads a Private Key from the PEM Encoded Store.
+ *
+ * @param basePath - Base Path, Directory where the Key is stored.
+ * @param keyFileName - File Name of the private key
+ * @return PrivateKey Object.
+ * @throws IOException - on Error.
+ */
+ private PKCS8EncodedKeySpec readKey(Path basePath, String keyFileName)
+ throws IOException {
+ File fileName = Paths.get(basePath.toString(), keyFileName).toFile();
+ String keyData = FileUtils.readFileToString(fileName, DEFAULT_CHARSET);
+ final byte[] pemContent;
+ try (PemReader pemReader = new PemReader(new StringReader(keyData))) {
+ PemObject keyObject = pemReader.readPemObject();
+ pemContent = keyObject.getContent();
+ }
+ return new PKCS8EncodedKeySpec(pemContent);
+ }
+
+ /**
+ * Returns a Private Key from a PEM encoded file.
+ *
+ * @param basePath - base path
+ * @param privateKeyFileName - private key file name.
+ * @return PrivateKey
+ * @throws InvalidKeySpecException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws IOException - on Error.
+ */
+ public PrivateKey readPrivateKey(Path basePath, String privateKeyFileName)
+ throws InvalidKeySpecException, NoSuchAlgorithmException, IOException {
+ PKCS8EncodedKeySpec encodedKeySpec = readKey(basePath, privateKeyFileName);
+ final KeyFactory keyFactory =
+ KeyFactory.getInstance(securityConfig.getProvider());
+ final PrivateKey privateKey =
+ keyFactory.generatePrivate(encodedKeySpec);
+ return privateKey;
+ }
+
+ /**
+ * Returns a public key from a PEM encoded file.
+ *
+ * @param basePath - base path.
+ * @param publicKeyFileName - public key file name.
+ * @return PublicKey
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws InvalidKeySpecException - on Error.
+ * @throws IOException - on Error.
+ */
+ public PublicKey readPublicKey(Path basePath, String publicKeyFileName)
+ throws NoSuchAlgorithmException, InvalidKeySpecException, IOException {
+ PKCS8EncodedKeySpec encodedKeySpec = readKey(basePath, publicKeyFileName);
+ final KeyFactory keyFactory =
+ KeyFactory.getInstance(securityConfig.getProvider());
+ final PublicKey publicKey =
+ keyFactory.generatePublic(encodedKeySpec);
+ return publicKey;
+ }
+
+ /**
+ * Helper function that actually writes data to the files.
+ *
+ * @param basePath - base path to write key
+ * @param keyPair - Key pair to write to file.
+ * @param privateKeyFileName - private key file name.
+ * @param publicKeyFileName - public key file name.
+ * @param force - forces overwriting the keys.
+ * @throws IOException - On I/O failure.
+ */
+ private synchronized void writeKey(Path basePath, KeyPair keyPair,
+ String privateKeyFileName, String publicKeyFileName, boolean force)
+ throws IOException {
+ checkPreconditions(basePath);
+
+ File privateKeyFile =
+ Paths.get(location.toString(), privateKeyFileName).toFile();
+ File publicKeyFile =
+ Paths.get(location.toString(), publicKeyFileName).toFile();
+ checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+ try (PemWriter privateKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+ privateKeyWriter.writeObject(
+ new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+ }
+
+ try (PemWriter publicKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+ publicKeyWriter.writeObject(
+ new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+ }
+ Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+ Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Checks if private and public key file already exists. Throws IOException if
+ * file exists and force flag is set to false, else will delete the existing
+ * file.
+ *
+ * @param privateKeyFile - Private key file.
+ * @param force - forces overwriting the keys.
+ * @param publicKeyFile - public key file.
+ * @throws IOException - On I/O failure.
+ */
+ private void checkKeyFile(File privateKeyFile, boolean force,
+ File publicKeyFile) throws IOException {
+ if (privateKeyFile.exists() && force) {
+ if (!privateKeyFile.delete()) {
+ throw new IOException("Unable to delete private key file.");
+ }
+ }
+
+ if (publicKeyFile.exists() && force) {
+ if (!publicKeyFile.delete()) {
+ throw new IOException("Unable to delete public key file.");
+ }
+ }
+
+ if (privateKeyFile.exists()) {
+ throw new IOException("Private Key file already exists.");
+ }
+
+ if (publicKeyFile.exists()) {
+ throw new IOException("Public Key file already exists.");
+ }
+ }
+
+ /**
+ * Checks if base path exists and sets file permissions.
+ *
+ * @param basePath - base path to write key
+ * @throws IOException - On I/O failure.
+ */
+ private void checkPreconditions(Path basePath) throws IOException {
+ Preconditions.checkNotNull(basePath, "Base path cannot be null");
+ if (!isPosixFileSystem.get()) {
+ LOG.error("Keys cannot be stored securely without POSIX file system "
+ + "support for now.");
+ throw new IOException("Unsupported File System for pem file.");
+ }
+
+ if (Files.exists(basePath)) {
+ // Not the end of the world if we reset the permissions on an existing
+ // directory.
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ } else {
+ boolean success = basePath.toFile().mkdirs();
+ if (!success) {
+ LOG.error("Unable to create the directory for the "
+ + "location. Location: {}", basePath);
+ throw new IOException("Unable to create the directory for the "
+ + "location. Location:" + basePath);
+ }
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
index 1cf39b2..ad26f77 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
@@ -62,7 +62,7 @@ public class StorageInfo {
* @param cT
* Cluster creation Time
- * @throws IOException
+ * @throws IOException - on Error.
*/
public StorageInfo(NodeType type, String cid, long cT)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
new file mode 100644
index 0000000..0e98ba7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.cert.CertificateException;
+import java.util.function.Consumer;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.junit.Assert.*;
+
+/**
+ * Tests the Default CA Server.
+ */
+public class TestDefaultCAServer {
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ }
+
+ @Test
+ public void testInit() throws SCMSecurityException, CertificateException,
+ IOException {
+ SecurityConfig securityConfig = new SecurityConfig(conf);
+ CertificateServer testCA = new DefaultCAServer("testCA",
+ RandomStringUtils.randomAlphabetic(4),
+ RandomStringUtils.randomAlphabetic(4));
+ testCA.init(securityConfig, CertificateServer.CAType.SELF_SIGNED_CA);
+ X509CertificateHolder first = testCA.getCACertificate();
+ assertNotNull(first);
+ //Init is idempotent.
+ testCA.init(securityConfig, CertificateServer.CAType.SELF_SIGNED_CA);
+ X509CertificateHolder second = testCA.getCACertificate();
+ assertEquals(first, second);
+
+ // we only support Self Signed CA for now.
+ try {
+ testCA.init(securityConfig, CertificateServer.CAType.INTERMEDIARY_CA);
+ fail("code should not reach here, exception should have been thrown.");
+ } catch (IllegalStateException e) {
+ // This is a run time exception, hence it is not caught by the junit
+ // expected Exception.
+ assertTrue(e.toString().contains("Not implemented"));
+ }
+ }
+
+ @Test
+ public void testMissingCertificate() {
+ SecurityConfig securityConfig = new SecurityConfig(conf);
+ CertificateServer testCA = new DefaultCAServer("testCA",
+ RandomStringUtils.randomAlphabetic(4),
+ RandomStringUtils.randomAlphabetic(4));
+ Consumer<SecurityConfig> caInitializer =
+ ((DefaultCAServer) testCA).processVerificationStatus(
+ DefaultCAServer.VerificationStatus.MISSING_CERTIFICATE);
+ try {
+
+ caInitializer.accept(securityConfig);
+ fail("code should not reach here, exception should have been thrown.");
+ } catch (IllegalStateException e) {
+ // This also is a runtime exception. Hence not caught by junit expected
+ // exception.
+ assertTrue(e.toString().contains("Missing Root Certs"));
+ }
+ }
+
+ @Test
+ public void testMissingKey() {
+ SecurityConfig securityConfig = new SecurityConfig(conf);
+ CertificateServer testCA = new DefaultCAServer("testCA",
+ RandomStringUtils.randomAlphabetic(4),
+ RandomStringUtils.randomAlphabetic(4));
+ Consumer<SecurityConfig> caInitializer =
+ ((DefaultCAServer) testCA).processVerificationStatus(
+ DefaultCAServer.VerificationStatus.MISSING_KEYS);
+ try {
+
+ caInitializer.accept(securityConfig);
+ fail("code should not reach here, exception should have been thrown.");
+ } catch (IllegalStateException e) {
+ // This also is a runtime exception. Hence not caught by junit expected
+ // exception.
+ assertTrue(e.toString().contains("Missing Keys"));
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
new file mode 100644
index 0000000..1d20a78
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Tests for Default CA.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
new file mode 100644
index 0000000..9ac956f
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCertificateCodec.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the Certificate codecs.
+ */
+public class TestCertificateCodec {
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ }
+
+ /**
+ * This test converts a X509Certificate Holder object to a PEM encoded String,
+ * then creates a new X509Certificate object to verify that we are able to
+ * serialize and deserialize correctly. we follow up with converting these
+ * objects to standard JCA x509Certificate objects.
+ *
+ * @throws NoSuchProviderException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ * @throws CertificateException - on Error.
+ */
+ @Test
+ public void testGetPEMEncodedString()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, SCMSecurityException, CertificateException {
+ HDDSKeyGenerator keyGenerator =
+ new HDDSKeyGenerator(conf);
+ X509CertificateHolder cert =
+ SelfSignedCertificate.newBuilder()
+ .setSubject(RandomStringUtils.randomAlphabetic(4))
+ .setClusterID(RandomStringUtils.randomAlphabetic(4))
+ .setScmID(RandomStringUtils.randomAlphabetic(4))
+ .setBeginDate(LocalDate.now())
+ .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+ .setConfiguration(keyGenerator.getSecurityConfig()
+ .getConfiguration())
+ .setKey(keyGenerator.generateKey())
+ .makeCA()
+ .build();
+ CertificateCodec codec = new CertificateCodec(conf);
+ String pemString = codec.getPEMEncodedString(cert);
+ assertTrue(pemString.startsWith(CertificateCodec.BEGIN_CERT));
+ assertTrue(pemString.endsWith(CertificateCodec.END_CERT + "\n"));
+
+ // Read back the certificate and verify that all the comparisons pass.
+ X509CertificateHolder newCert =
+ codec.getCertificateHolder(codec.getX509Certificate(pemString));
+ assertEquals(cert, newCert);
+
+ // Just make sure we can decode both these classes to Java Std. lIb classes.
+ X509Certificate firstCert = CertificateCodec.getX509Certificate(cert);
+ X509Certificate secondCert = CertificateCodec.getX509Certificate(newCert);
+ assertEquals(firstCert, secondCert);
+ }
+
+ /**
+ * tests writing and reading certificates in PEM encoded form.
+ *
+ * @throws NoSuchProviderException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ * @throws CertificateException - on Error.
+ */
+ @Test
+ public void testwriteCertificate() throws NoSuchProviderException,
+ NoSuchAlgorithmException, IOException, SCMSecurityException,
+ CertificateException {
+ HDDSKeyGenerator keyGenerator =
+ new HDDSKeyGenerator(conf);
+ X509CertificateHolder cert =
+ SelfSignedCertificate.newBuilder()
+ .setSubject(RandomStringUtils.randomAlphabetic(4))
+ .setClusterID(RandomStringUtils.randomAlphabetic(4))
+ .setScmID(RandomStringUtils.randomAlphabetic(4))
+ .setBeginDate(LocalDate.now())
+ .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+ .setConfiguration(keyGenerator.getSecurityConfig()
+ .getConfiguration())
+ .setKey(keyGenerator.generateKey())
+ .makeCA()
+ .build();
+ CertificateCodec codec = new CertificateCodec(conf);
+ String pemString = codec.getPEMEncodedString(cert);
+ File basePath = temporaryFolder.newFolder();
+ if (!basePath.exists()) {
+ Assert.assertTrue(basePath.mkdirs());
+ }
+ codec.writeCertificate(basePath.toPath(), "pemcertificate.crt",
+ pemString, false);
+ X509CertificateHolder certHolder =
+ codec.readCertificate(basePath.toPath(), "pemcertificate.crt");
+ assertNotNull(certHolder);
+ assertEquals(cert.getSerialNumber(), certHolder.getSerialNumber());
+ }
+
+ /**
+ * Tests reading and writing certificates in DER form.
+ *
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ * @throws CertificateException - on Error.
+ * @throws NoSuchProviderException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ */
+ @Test
+ public void testwriteCertificateDefault()
+ throws IOException, SCMSecurityException, CertificateException,
+ NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGenerator =
+ new HDDSKeyGenerator(conf);
+ X509CertificateHolder cert =
+ SelfSignedCertificate.newBuilder()
+ .setSubject(RandomStringUtils.randomAlphabetic(4))
+ .setClusterID(RandomStringUtils.randomAlphabetic(4))
+ .setScmID(RandomStringUtils.randomAlphabetic(4))
+ .setBeginDate(LocalDate.now())
+ .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+ .setConfiguration(keyGenerator.getSecurityConfig()
+ .getConfiguration())
+ .setKey(keyGenerator.generateKey())
+ .makeCA()
+ .build();
+ CertificateCodec codec = new CertificateCodec(conf);
+ codec.writeCertificate(cert);
+ X509CertificateHolder certHolder = codec.readCertificate();
+ assertNotNull(certHolder);
+ assertEquals(cert.getSerialNumber(), certHolder.getSerialNumber());
+ }
+
+ /**
+ * Tests writing to non-default certificate file name.
+ *
+ * @throws IOException - on Error.
+ * @throws SCMSecurityException - on Error.
+ * @throws NoSuchProviderException - on Error.
+ * @throws NoSuchAlgorithmException - on Error.
+ * @throws CertificateException - on Error.
+ */
+ @Test
+ public void writeCertificate2() throws IOException, SCMSecurityException,
+ NoSuchProviderException, NoSuchAlgorithmException, CertificateException {
+ HDDSKeyGenerator keyGenerator =
+ new HDDSKeyGenerator(conf);
+ X509CertificateHolder cert =
+ SelfSignedCertificate.newBuilder()
+ .setSubject(RandomStringUtils.randomAlphabetic(4))
+ .setClusterID(RandomStringUtils.randomAlphabetic(4))
+ .setScmID(RandomStringUtils.randomAlphabetic(4))
+ .setBeginDate(LocalDate.now())
+ .setEndDate(LocalDate.now().plus(1, ChronoUnit.DAYS))
+ .setConfiguration(keyGenerator.getSecurityConfig()
+ .getConfiguration())
+ .setKey(keyGenerator.generateKey())
+ .makeCA()
+ .build();
+ CertificateCodec codec =
+ new CertificateCodec(keyGenerator.getSecurityConfig(), "ca");
+ codec.writeCertificate(cert, "newcert.crt", false);
+ // Rewrite with force support
+ codec.writeCertificate(cert, "newcert.crt", true);
+ X509CertificateHolder x509CertificateHolder =
+ codec.readCertificate(codec.getLocation(), "newcert.crt");
+ assertNotNull(x509CertificateHolder);
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 9328c50..d234b66 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,20 +6,22 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
-package org.apache.hadoop.hdds.security.x509.certificates;
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.bouncycastle.asn1.ASN1Sequence;
@@ -45,12 +47,15 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+/**
+ * Certificate Signing Request.
+ */
public class TestCertificateSignRequest {
- private SecurityConfig securityConfig;
private static OzoneConfiguration conf = new OzoneConfiguration();
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private SecurityConfig securityConfig;
@Before
public void init() throws IOException {
@@ -257,7 +262,7 @@ public class TestCertificateSignRequest {
// Verify CSR with attribute for extensions
Assert.assertEquals(1, csr.getAttributes().length);
}
-
+
@Test
public void testCsrSerialization() throws NoSuchProviderException,
NoSuchAlgorithmException, SCMSecurityException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index e581dc8..416cc61 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -5,7 +5,7 @@
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -17,11 +17,12 @@
*
*/
-package org.apache.hadoop.hdds.security.x509.certificates;
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.cert.X509CertificateHolder;
@@ -41,8 +42,8 @@ import java.security.NoSuchProviderException;
import java.security.SignatureException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
-import java.time.Duration;
-import java.time.Instant;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.UUID;
@@ -52,10 +53,10 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
* Test Class for Root Certificate generation.
*/
public class TestRootCertificate {
- private SecurityConfig securityConfig;
private static OzoneConfiguration conf = new OzoneConfiguration();
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private SecurityConfig securityConfig;
@Before
public void init() throws IOException {
@@ -67,10 +68,9 @@ public class TestRootCertificate {
public void testAllFieldsAreExpected()
throws SCMSecurityException, NoSuchProviderException,
NoSuchAlgorithmException, CertificateException,
- SignatureException, InvalidKeyException {
- Instant now = Instant.now();
- Date notBefore = Date.from(now);
- Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ SignatureException, InvalidKeyException, IOException {
+ LocalDate notBefore = LocalDate.now();
+ LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
String clusterID = UUID.randomUUID().toString();
String scmID = UUID.randomUUID().toString();
String subject = "testRootCert";
@@ -96,13 +96,15 @@ public class TestRootCertificate {
// Make sure that NotBefore is before the current Date
- Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+ Date invalidDate = java.sql.Date.valueOf(
+ notBefore.minus(1, ChronoUnit.DAYS));
Assert.assertFalse(
certificateHolder.getNotBefore()
.before(invalidDate));
//Make sure the end date is honored.
- invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+ invalidDate = java.sql.Date.valueOf(
+ notAfter.plus(1, ChronoUnit.DAYS));
Assert.assertFalse(
certificateHolder.getNotAfter()
.after(invalidDate));
@@ -113,7 +115,8 @@ public class TestRootCertificate {
Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
- // We did not ask for this Certificate to be a CA certificate, hence that
+ // We did not ask for this Certificate to be a CertificateServer
+ // certificate, hence that
// extension should be null.
Assert.assertNull(
certificateHolder.getExtension(Extension.basicConstraints));
@@ -128,10 +131,9 @@ public class TestRootCertificate {
@Test
public void testCACert()
throws SCMSecurityException, NoSuchProviderException,
- NoSuchAlgorithmException {
- Instant now = Instant.now();
- Date notBefore = Date.from(now);
- Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ NoSuchAlgorithmException, IOException {
+ LocalDate notBefore = LocalDate.now();
+ LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
String clusterID = UUID.randomUUID().toString();
String scmID = UUID.randomUUID().toString();
String subject = "testRootCert";
@@ -151,7 +153,8 @@ public class TestRootCertificate {
.makeCA();
X509CertificateHolder certificateHolder = builder.build();
- // This time we asked for a CA Certificate, make sure that extension is
+ // This time we asked for a CertificateServer Certificate, make sure that
+ // extension is
// present and valid.
Extension basicExt =
certificateHolder.getExtension(Extension.basicConstraints);
@@ -167,10 +170,9 @@ public class TestRootCertificate {
@Test
public void testInvalidParamFails()
throws SCMSecurityException, NoSuchProviderException,
- NoSuchAlgorithmException {
- Instant now = Instant.now();
- Date notBefore = Date.from(now);
- Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ NoSuchAlgorithmException, IOException {
+ LocalDate notBefore = LocalDate.now();
+ LocalDate notAfter = notBefore.plus(365, ChronoUnit.DAYS);
String clusterID = UUID.randomUUID().toString();
String scmID = UUID.randomUUID().toString();
String subject = "testRootCert";
@@ -253,6 +255,4 @@ public class TestRootCertificate {
// Assert that we can create a certificate with all sane params.
Assert.assertNotNull(builder.build());
}
-
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
index c8a31fc..fffe1e5 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -5,7 +5,7 @@
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -19,4 +19,4 @@
/**
* Test classes for Certificate utilities.
*/
-package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
+package org.apache.hadoop.hdds.security.x509.certificate.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
deleted file mode 100644
index db5d430..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.keys;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test class for HDDS pem writer.
- */
-public class TestHDDSKeyPEMWriter {
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private OzoneConfiguration configuration;
- private HDDSKeyGenerator keyGenerator;
- private String prefix;
-
- @Before
- public void init() throws IOException {
- configuration = new OzoneConfiguration();
- prefix = temporaryFolder.newFolder().toString();
- configuration.set(HDDS_METADATA_DIR_NAME, prefix);
- keyGenerator = new HDDSKeyGenerator(configuration);
- }
-
- /**
- * Assert basic things like we are able to create a file, and the names are
- * in expected format etc.
- *
- * @throws NoSuchProviderException - On Error, due to missing Java
- * dependencies.
- * @throws NoSuchAlgorithmException - On Error, due to missing Java
- * dependencies.
- * @throws IOException - On I/O failure.
- */
- @Test
- public void testWriteKey()
- throws NoSuchProviderException, NoSuchAlgorithmException,
- IOException, InvalidKeySpecException {
- KeyPair keys = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.writeKey(keys);
-
- // Assert that locations have been created.
- Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
- Assert.assertTrue(keyLocation.toFile().exists());
-
- // Assert that locations are created in the locations that we specified
- // using the Config.
- Assert.assertTrue(keyLocation.toString().startsWith(prefix));
- Path privateKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPrivateKeyFileName());
- Assert.assertTrue(privateKeyPath.toFile().exists());
- Path publicKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPublicKeyFileName());
- Assert.assertTrue(publicKeyPath.toFile().exists());
-
- // Read the private key and test if the expected String in the PEM file
- // format exists.
- byte[] privateKey = Files.readAllBytes(privateKeyPath);
- String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
- Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
-
- // Read the public key and test if the expected String in the PEM file
- // format exists.
- byte[] publicKey = Files.readAllBytes(publicKeyPath);
- String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
- Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
-
- // Let us decode the PEM file and parse it back into binary.
- KeyFactory kf = KeyFactory.getInstance(
- pemWriter.getSecurityConfig().getKeyAlgo());
-
- // Replace the PEM Human readable guards.
- privateKeydata =
- privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
- privateKeydata =
- privateKeydata.replace("-----END PRIVATE KEY-----", "");
-
- // Decode the bas64 to binary format and then use an ASN.1 parser to
- // parse the binary format.
-
- byte[] keyBytes = Base64.decodeBase64(privateKeydata);
- PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
- PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
- Assert.assertNotNull("Private Key should not be null",
- privateKeyDecoded);
-
- // Let us decode the public key and veriy that we can parse it back into
- // binary.
- publicKeydata =
- publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
- publicKeydata =
- publicKeydata.replace("-----END PUBLIC KEY-----", "");
-
- keyBytes = Base64.decodeBase64(publicKeydata);
- X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
- PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
- Assert.assertNotNull("Public Key should not be null",
- publicKeyDecoded);
-
- // Now let us assert the permissions on the Directories and files are as
- // expected.
- Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
- Set<PosixFilePermission> currentSet =
- Files.getPosixFilePermissions(privateKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(publicKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(keyLocation);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
- }
-
- /**
- * Assert key rewrite fails without force option.
- *
- * @throws IOException - on I/O failure.
- */
- @Test
- public void testReWriteKey()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- SecurityConfig secConfig = pemWriter.getSecurityConfig();
- pemWriter.writeKey(kp);
-
- // Assert that rewriting of keys throws exception with valid messages.
- LambdaTestUtils
- .intercept(IOException.class, "Private Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPrivateKeyFileName()).toFile());
- LambdaTestUtils
- .intercept(IOException.class, "Public Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPublicKeyFileName()).toFile());
-
- // Should succeed now as both public and private key are deleted.
- pemWriter.writeKey(kp);
- // Should succeed with overwrite flag as true.
- pemWriter.writeKey(kp, true);
-
- }
-
- /**
- * Assert key rewrite fails in non Posix file system.
- *
- * @throws IOException - on I/O failure.
- */
- @Test
- public void testWriteKeyInNonPosixFS()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.setIsPosixFileSystem(() -> false);
-
- // Assert key rewrite fails in non Posix file system.
- LambdaTestUtils
- .intercept(IOException.class, "Unsupported File System for pem file.",
- () -> pemWriter.writeKey(kp));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
new file mode 100644
index 0000000..f0973f7
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestKeyCodec.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestKeyCodec {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OzoneConfiguration configuration;
+ private HDDSKeyGenerator keyGenerator;
+ private String prefix;
+
+ @Before
+ public void init() throws IOException {
+ configuration = new OzoneConfiguration();
+ prefix = temporaryFolder.newFolder().toString();
+ configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+ keyGenerator = new HDDSKeyGenerator(configuration);
+ }
+
+ /**
+ * Assert basic things like we are able to create a file, and the names are
+ * in expected format etc.
+ *
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ * @throws IOException - On I/O failure.
+ */
+ @Test
+ public void testWriteKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, InvalidKeySpecException {
+ KeyPair keys = keyGenerator.generateKey();
+ KeyCodec pemWriter = new KeyCodec(configuration);
+ pemWriter.writeKey(keys);
+
+ // Assert that locations have been created.
+ Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+ Assert.assertTrue(keyLocation.toFile().exists());
+
+ // Assert that locations are created in the locations that we specified
+ // using the Config.
+ Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+ Path privateKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPrivateKeyFileName());
+ Assert.assertTrue(privateKeyPath.toFile().exists());
+ Path publicKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPublicKeyFileName());
+ Assert.assertTrue(publicKeyPath.toFile().exists());
+
+ // Read the private key and test if the expected String in the PEM file
+ // format exists.
+ byte[] privateKey = Files.readAllBytes(privateKeyPath);
+ String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+ // Read the public key and test if the expected String in the PEM file
+ // format exists.
+ byte[] publicKey = Files.readAllBytes(publicKeyPath);
+ String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+ // Let us decode the PEM file and parse it back into binary.
+ KeyFactory kf = KeyFactory.getInstance(
+ pemWriter.getSecurityConfig().getKeyAlgo());
+
+ // Replace the PEM Human readable guards.
+ privateKeydata =
+ privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+ privateKeydata =
+ privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+ // Decode the bas64 to binary format and then use an ASN.1 parser to
+ // parse the binary format.
+
+ byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+ PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+ PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+ Assert.assertNotNull("Private Key should not be null",
+ privateKeyDecoded);
+
+ // Let us decode the public key and veriy that we can parse it back into
+ // binary.
+ publicKeydata =
+ publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+ publicKeydata =
+ publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+ keyBytes = Base64.decodeBase64(publicKeydata);
+ X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+ PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+ Assert.assertNotNull("Public Key should not be null",
+ publicKeyDecoded);
+
+ // Now let us assert the permissions on the Directories and files are as
+ // expected.
+ Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+ Set<PosixFilePermission> currentSet =
+ Files.getPosixFilePermissions(privateKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(publicKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(keyLocation);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+ }
+
+ /**
+ * Assert key rewrite fails without force option.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testReWriteKey()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ KeyCodec pemWriter = new KeyCodec(configuration);
+ SecurityConfig secConfig = pemWriter.getSecurityConfig();
+ pemWriter.writeKey(kp);
+
+ // Assert that rewriting of keys throws exception with valid messages.
+ LambdaTestUtils
+ .intercept(IOException.class, "Private Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPrivateKeyFileName()).toFile());
+ LambdaTestUtils
+ .intercept(IOException.class, "Public Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPublicKeyFileName()).toFile());
+
+ // Should succeed now as both public and private key are deleted.
+ pemWriter.writeKey(kp);
+ // Should succeed with overwrite flag as true.
+ pemWriter.writeKey(kp, true);
+
+ }
+
+ /**
+ * Assert key rewrite fails in non Posix file system.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testWriteKeyInNonPosixFS()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ KeyCodec pemWriter = new KeyCodec(configuration);
+ pemWriter.setIsPosixFileSystem(() -> false);
+
+ // Assert key rewrite fails in non Posix file system.
+ LambdaTestUtils
+ .intercept(IOException.class, "Unsupported File System for pem file.",
+ () -> pemWriter.writeKey(kp));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
index 47ad597..d5695c5 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
@@ -31,7 +31,6 @@ import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
/**
* Tests RDBStore creation.
@@ -131,7 +130,7 @@ public class TestDBStoreBuilder {
RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
firstTable.put(key, value);
byte[] temp = firstTable.get(key);
- Assert.assertTrue(Arrays.equals(value, temp));
+ Assert.assertArrayEquals(value, temp);
}
try (Table secondTable = dbStore.getTable("Second")) {
@@ -161,7 +160,7 @@ public class TestDBStoreBuilder {
RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
firstTable.put(key, value);
byte[] temp = firstTable.get(key);
- Assert.assertTrue(Arrays.equals(value, temp));
+ Assert.assertArrayEquals(value, temp);
}
try (Table secondTable = dbStore.getTable("Second")) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f51d23/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index c1695f1..7fb995e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
-import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyPEMWriter;
+import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.RPC;
@@ -86,7 +86,7 @@ public final class TestSecureOzoneCluster {
private static final String TEST_USER = "testUgiUser";
private static final int CLIENT_TIMEOUT = 2 * 1000;
- private Logger LOGGER = LoggerFactory
+ private Logger logger = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
@Rule
@@ -125,9 +125,9 @@ public final class TestSecureOzoneCluster {
createCredentialsInKDC(conf, miniKdc);
generateKeyPair(conf);
} catch (IOException e) {
- LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ logger.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
- LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ logger.error("Failed to initialize TestSecureOzoneCluster", e);
}
}
@@ -146,7 +146,7 @@ public final class TestSecureOzoneCluster {
}
FileUtils.deleteQuietly(metaDirPath.toFile());
} catch (Exception e) {
- LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+ logger.error("Failed to stop TestSecureOzoneCluster", e);
}
}
@@ -449,7 +449,7 @@ public final class TestSecureOzoneCluster {
private void generateKeyPair(OzoneConfiguration config) throws Exception {
HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf);
keyPair = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(config);
+ KeyCodec pemWriter = new KeyCodec(config);
pemWriter.writeKey(keyPair, true);
}
@@ -460,8 +460,6 @@ public final class TestSecureOzoneCluster {
*/
@Test
public void testDelegationTokenRenewal() throws Exception {
- // Capture logs for assertions.
- LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
GenericTestUtils
.setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
@@ -477,7 +475,6 @@ public final class TestSecureOzoneCluster {
om.start();
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String username = ugi.getUserName();
// Get first OM client which will authenticate via Kerberos
omClient = new OzoneManagerProtocolClientSideTranslatorPB(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8948. PlacementRule interface
should be for all YarnSchedulers. Contributed by Bibin A Chundatt.
Posted by xy...@apache.org.
YARN-8948. PlacementRule interface should be for all YarnSchedulers. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a68d766e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a68d766e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a68d766e
Branch: refs/heads/HDDS-4
Commit: a68d766e876631d7ee2e1a6504d4120ba628d178
Parents: c1d24f8
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 21:43:34 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 21:43:34 2018 +0530
----------------------------------------------------------------------
.../placement/AppNameMappingPlacementRule.java | 12 ++++++++++--
.../server/resourcemanager/placement/PlacementRule.java | 4 ++--
.../placement/UserGroupMappingPlacementRule.java | 11 ++++++++++-
3 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
index 2debade..7a46962 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -61,8 +62,15 @@ public class AppNameMappingPlacementRule extends PlacementRule {
}
@Override
- public boolean initialize(CapacitySchedulerContext schedulerContext)
+ public boolean initialize(ResourceScheduler scheduler)
throws IOException {
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new IOException(
+ "AppNameMappingPlacementRule can be configured only for "
+ + "CapacityScheduler");
+ }
+ CapacitySchedulerContext schedulerContext =
+ (CapacitySchedulerContext) scheduler;
CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
LOG.info(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
index 21ab32a..0f3d43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
public abstract class PlacementRule {
@@ -31,7 +31,7 @@ public abstract class PlacementRule {
}
public abstract boolean initialize(
- CapacitySchedulerContext schedulerContext) throws IOException;
+ ResourceScheduler scheduler) throws IOException;
/**
* Get queue for a given application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 99f37b1..a9e8235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -34,8 +34,10 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -230,8 +232,15 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
@VisibleForTesting
@Override
- public boolean initialize(CapacitySchedulerContext schedulerContext)
+ public boolean initialize(ResourceScheduler scheduler)
throws IOException {
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new IOException(
+ "UserGroupMappingPlacementRule can be configured only for "
+ + "CapacityScheduler");
+ }
+ CapacitySchedulerContext schedulerContext =
+ (CapacitySchedulerContext) scheduler;
CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
LOG.info(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDDS-10. Add kdc docker image for
secure ozone cluster. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-10. Add kdc docker image for secure ozone cluster. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/812c07ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/812c07ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/812c07ee
Branch: refs/heads/HDDS-4
Commit: 812c07ee4ffe110953d0d6b4ff3c1cb0287d569b
Parents: 585c344
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 4 13:20:09 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../dist/src/main/compose/ozonesecure/README.md | 22 +++++
.../compose/ozonesecure/docker-compose.yaml | 94 ++++++++++++--------
.../docker-image/docker-krb5/Dockerfile-krb5 | 33 +++++++
.../docker-image/docker-krb5/README.md | 34 +++++++
.../docker-image/docker-krb5/kadm5.acl | 1 +
.../docker-image/docker-krb5/krb5.conf | 40 +++++++++
.../docker-image/docker-krb5/launcher.sh | 25 ++++++
7 files changed, 210 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
new file mode 100644
index 0000000..0ce9a0a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+#### Dockerfile for KDC:
+* ./docker-image/docker-krb5/Dockerfile-krb5
+
+#### Dockerfile for SCM,OM and DataNode:
+* ./docker-image/runner/Dockerfile
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 42ab05e..fab5ba9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -16,42 +16,58 @@
version: "3"
services:
- kdc:
- image: ahadoop/kdc:v1
- hostname: kdc
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- datanode:
- image: ahadoop/runner:latest
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- hostname: datanode
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: ahadoop/runner:latest
- hostname: om
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: ahadoop/runner:latest
- hostname: scm
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
+ kdc:
+ build:
+ context: docker-image/docker-krb5
+ dockerfile: Dockerfile-krb5
+ args:
+ buildno: 1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ om:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
new file mode 100644
index 0000000..b5b931d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+
+FROM frolvlad/alpine-oraclejdk8:slim
+RUN apk add --update bash ca-certificates openssl krb5-server krb5 && rm -rf /var/cache/apk/* && update-ca-certificates
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN wget -O /root/issuer https://github.com/ajayydv/docker/raw/kdc/issuer
+RUN chmod +x /root/issuer
+WORKDIR /opt
+ADD krb5.conf /etc/
+ADD kadm5.acl /var/lib/krb5kdc/kadm5.acl
+RUN kdb5_util create -s -P Welcome1
+RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
+RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
+ADD launcher.sh .
+RUN mkdir -p /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
new file mode 100644
index 0000000..b864a5f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
@@ -0,0 +1,34 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+The docker image contains a rest service which provides keystore and keytab files without any authentication!
+
+Master password: Welcome1
+
+Principal: admin/admin@EXAMPLE.COM Password: Welcome1
+
+Test:
+
+```
+docker run --net=host krb5
+
+docker run --net=host -it --entrypoint=bash krb5
+kinit admin/admin
+#pwd: Welcome1
+klist
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
new file mode 100644
index 0000000..8fe9f69
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -0,0 +1 @@
+*/admin@EXAMPLE.COM x
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
new file mode 100644
index 0000000..34eabe1
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+default = FILE:/var/log/krb5libs.log
+kdc = FILE:/var/log/krb5kdc.log
+admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = localhost
+ admin_server = localhost
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/812c07ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
new file mode 100644
index 0000000..0824f7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+/root/issuer &
+krb5kdc -n &
+sleep 4
+kadmind -nofork &
+sleep 2
+tail -f /var/log/krb5kdc.log &
+tail -f /var/log/kadmind.log
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-592. Fix ozone-secure.robot
test. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bbc95ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bbc95ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bbc95ee
Branch: refs/heads/HDDS-4
Commit: 8bbc95ee536caae21f28f9513fe624955aa122a0
Parents: 9128c66
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 6 16:53:04 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../acceptance/ozone-secure.robot | 95 ----------------
.../dist/src/main/compose/ozonesecure/.env | 1 -
.../compose/ozonesecure/docker-compose.yaml | 22 ++--
.../ozonesecure/docker-image/runner/Dockerfile | 4 +-
.../main/smoketest/security/ozone-secure.robot | 111 +++++++++++++++++++
hadoop-ozone/dist/src/main/smoketest/test.sh | 2 +
6 files changed, 126 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
deleted file mode 100644
index 7fc1088..0000000
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Smoke test to start cluster with docker-compose environments.
-Library OperatingSystem
-Suite Setup Startup Ozone Cluster
-Suite Teardown Teardown Ozone Cluster
-
-*** Variables ***
-${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${version}
-
-*** Test Cases ***
-
-Daemons are running
- Is daemon running om
- Is daemon running scm
- Is daemon running datanode
- Is daemon running ozone.kdc
-
-Check if datanode is connected to the scm
- Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
-
-Test rest interface
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 200 OK
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 200 OK
-
-Test ozone cli
- ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- Should contain ${result} Client cannot authenticate via
- # Authenticate testuser
- Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
- Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
- Should contain ${result} createdOn
- Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
- ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
- Should Be Equal ${result} bill
-
-*** Keywords ***
-
-Startup Ozone Cluster
- ${rc} ${output} = Run docker compose 0 down
- ${rc} ${output} = Run docker compose 0 up -d
- Should Be Equal As Integers ${rc} 0
- Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening
-
-Teardown Ozone Cluster
- Run docker compose 0 down
-
-Is daemon running
- [arguments] ${name}
- ${result} = Run docker ps
- Should contain ${result} _${name}_1
-
-Is Daemon started
- [arguments] ${name} ${expression}
- ${rc} ${result} = Run docker compose 0 logs
- Should contain ${result} ${expression}
-
-Have healthy datanodes
- [arguments] ${requirednodes}
- ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
- Should Be Equal ${result} ${requirednodes}
-
-Execute on
- [arguments] ${expected_rc} ${componentname} ${command}
- ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command}
- [return] ${return}
-
-Run docker compose
- [arguments] ${expected_rc} ${command}
- Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone
- ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
- Should Be Equal As Integers ${rc} ${expected_rc}
- [return] ${rc} ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index a494004..cac418a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,4 +15,3 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index fab5ba9..034d0ff 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
buildno: 1
hostname: kdc
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
datanode:
build:
context: docker-image/runner
@@ -32,13 +32,13 @@ services:
args:
buildno: 1
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
hostname: datanode
ports:
- - 9864
+ - 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- - docker-config
+ - docker-config
om:
build:
context: docker-image/runner
@@ -47,13 +47,13 @@ services:
buildno: 1
hostname: om
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9874:9874
+ - 9874:9874
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
- - docker-config
+ - docker-config
command: ["/opt/hadoop/bin/ozone","om"]
scm:
build:
@@ -63,11 +63,11 @@ services:
buildno: 1
hostname: scm
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9876:9876
+ - 9876:9876
env_file:
- - docker-config
+ - docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
+ command: ["/opt/hadoop/bin/ozone","scm"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
index efda03e..17eff18 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -15,8 +15,8 @@
# limitations under the License.
FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
-
+RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean
+RUN pip install robotframework
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
RUN chmod +x /usr/local/bin/dumb-init
RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
new file mode 100644
index 0000000..457ebaa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test to start cluster with docker-compose environments.
+Library OperatingSystem
+Resource ../commonlib.robot
+
+*** Test Cases ***
+Create volume and bucket
+ ${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Should contain ${output} Client cannot authenticate via
+ # Authenticate testuser
+ Execute kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+ Execute ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Execute ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root
+ Execute ozone sh bucket create o3://om/fstest/bucket1
+ Execute ozone sh bucket create o3://om/fstest/bucket2
+ Execute ozone sh bucket create o3://om/fstest2/bucket3
+
+Check volume from ozonefs
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
+
+Run ozoneFS tests
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
+ Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} PUTFILE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+ Should contain ${result} NOTICE.txt
+ Should contain ${result} PUTFILE.txt
+
+ Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} MOVED.TXT
+ Should not contain ${result} NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} subdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ Should not contain ${result} Failed
+
+ Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} NOTICE.txt
+
+ ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} subdir1
+
+ Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} TOUCHFILE.txt
+
+ Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} testdir
+
+ Execute rm -Rf localdir1
+ Execute mkdir localdir1
+ Execute cp NOTICE.txt localdir1/LOCAL.txt
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
+ Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
+ Should contain ${result} localdir1/LOCAL.txt
+ Should contain ${result} testdir1/NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
+ Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
+ Execute ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt
+ Should contain ${result} KEY.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} File exists
+ Execute rm -Rf GET.txt
+ Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
+ Execute ls -l GET.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} VOLUME_NOT_FOUND
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
index 5819595..53d0876 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then
#
# We select the test suites and execute them on multiple type of clusters
#
+ DEFAULT_TESTS=("security")
+ execute_tests ozonesecure "${DEFAULT_TESTS[@]}"
DEFAULT_TESTS=("basic")
execute_tests ozone "${DEFAULT_TESTS[@]}"
TESTS=("ozonefs")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-877. Ensure correct surefire
version for Ozone test. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-877. Ensure correct surefire version for Ozone test. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae5fbdd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae5fbdd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae5fbdd9
Branch: refs/heads/HDDS-4
Commit: ae5fbdd9ed6ef09b588637f2eadd7a04e8382289
Parents: f534736
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Nov 29 11:37:36 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:37:36 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/pom.xml | 1 +
hadoop-ozone/pom.xml | 3 +--
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 869ecbf..5537b3a 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -53,6 +53,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+ <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
</properties>
<repositories>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 39c65d5..4c13bd6 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -37,8 +37,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<bouncycastle.version>1.60</bouncycastle.version>
<ozone.release>Badlands</ozone.release>
<declared.ozone.version>${ozone.version}</declared.ozone.version>
-
-
+ <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
</properties>
<modules>
<module>common</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-15959. Revert
"HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@'
to be non-simple"
Posted by xy...@apache.org.
HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"
This reverts commit 829a2e4d271f05afb209ddc834cd4a0e85492eda.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0edd372
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0edd372
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0edd372
Branch: refs/heads/HDDS-4
Commit: d0edd37269bb40290b409d583bcf3b70897c13e0
Parents: 5e102f9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 29 17:52:11 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 17:52:11 2018 +0000
----------------------------------------------------------------------
.../authentication/util/KerberosName.java | 9 ++--
.../TestKerberosAuthenticationHandler.java | 7 ++-
.../authentication/util/TestKerberosName.java | 17 ++++++--
.../java/org/apache/hadoop/security/KDiag.java | 46 +-------------------
.../src/site/markdown/SecureMode.md | 6 ---
.../org/apache/hadoop/security/TestKDiag.java | 16 -------
.../security/TestUserGroupInformation.java | 27 ++++--------
7 files changed, 33 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
index 4e7ee3c..287bb13 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
@@ -324,8 +324,8 @@ public class KerberosName {
}
}
if (result != null && nonSimplePattern.matcher(result).find()) {
- LOG.info("Non-simple name {} after auth_to_local rule {}",
- result, this);
+ throw new NoMatchingRule("Non-simple name " + result +
+ " after auth_to_local rule " + this);
}
if (toLowerCase && result != null) {
result = result.toLowerCase(Locale.ENGLISH);
@@ -378,7 +378,7 @@ public class KerberosName {
/**
* Get the translation of the principal name into an operating system
* user name.
- * @return the user name
+ * @return the short name
* @throws IOException throws if something is wrong with the rules
*/
public String getShortName() throws IOException {
@@ -398,8 +398,7 @@ public class KerberosName {
return result;
}
}
- LOG.info("No auth_to_local rules applied to {}", this);
- return toString();
+ throw new NoMatchingRule("No rules applied to " + toString());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
index e672391..8b4bc15 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
@@ -108,7 +108,12 @@ public class TestKerberosAuthenticationHandler
kn = new KerberosName("bar@BAR");
Assert.assertEquals("bar", kn.getShortName());
kn = new KerberosName("bar@FOO");
- Assert.assertEquals("bar@FOO", kn.getShortName());
+ try {
+ kn.getShortName();
+ Assert.fail();
+ }
+ catch (Exception ex) {
+ }
}
@Test(timeout=60000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
index c584fce..2db0df4 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
@@ -72,14 +72,23 @@ public class TestKerberosName {
}
}
+ private void checkBadTranslation(String from) {
+ System.out.println("Checking bad translation for " + from);
+ KerberosName nm = new KerberosName(from);
+ try {
+ nm.getShortName();
+ Assert.fail("didn't get exception for " + from);
+ } catch (IOException ie) {
+ // PASS
+ }
+ }
+
@Test
public void testAntiPatterns() throws Exception {
checkBadName("owen/owen/owen@FOO.COM");
checkBadName("owen@foo/bar.com");
-
- // no rules applied, these should pass
- checkTranslation("foo@ACME.COM", "foo@ACME.COM");
- checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM");
+ checkBadTranslation("foo@ACME.COM");
+ checkBadTranslation("root/joe@FOO.COM");
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index b4e535c..1711921 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -22,7 +22,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
@@ -55,7 +54,6 @@ import java.util.Collections;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
-import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
import static org.apache.hadoop.security.UserGroupInformation.*;
@@ -131,12 +129,6 @@ public class KDiag extends Configured implements Tool, Closeable {
private boolean nofail = false;
private boolean nologin = false;
private boolean jaas = false;
- private boolean checkShortName = false;
-
- /**
- * A pattern that recognizes simple/non-simple names. Per KerberosName
- */
- private static final Pattern nonSimplePattern = Pattern.compile("[/@]");
/**
* Flag set to true if a {@link #verify(boolean, String, String, Object...)}
@@ -165,8 +157,6 @@ public class KDiag extends Configured implements Tool, Closeable {
public static final String ARG_SECURE = "--secure";
- public static final String ARG_VERIFYSHORTNAME = "--verifyshortname";
-
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public KDiag(Configuration conf,
PrintWriter out,
@@ -210,7 +200,6 @@ public class KDiag extends Configured implements Tool, Closeable {
nofail = popOption(ARG_NOFAIL, args);
jaas = popOption(ARG_JAAS, args);
nologin = popOption(ARG_NOLOGIN, args);
- checkShortName = popOption(ARG_VERIFYSHORTNAME, args);
// look for list of resources
String resource;
@@ -256,9 +245,7 @@ public class KDiag extends Configured implements Tool, Closeable {
+ arg(ARG_NOLOGIN, "", "Do not attempt to log in")
+ arg(ARG_OUTPUT, "<file>", "Write output to a file")
+ arg(ARG_RESOURCE, "<resource>", "Load an XML configuration resource")
- + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure")
- + arg(ARG_VERIFYSHORTNAME, ARG_PRINCIPAL + " <principal>",
- "Verify the short name of the specific principal does not contain '@' or '/'");
+ + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure");
}
private String arg(String name, String params, String meaning) {
@@ -291,7 +278,6 @@ public class KDiag extends Configured implements Tool, Closeable {
println("%s = %d", ARG_KEYLEN, minKeyLength);
println("%s = %s", ARG_KEYTAB, keytab);
println("%s = %s", ARG_PRINCIPAL, principal);
- println("%s = %s", ARG_VERIFYSHORTNAME, checkShortName);
// Fail fast on a JVM without JCE installed.
validateKeyLength();
@@ -391,10 +377,6 @@ public class KDiag extends Configured implements Tool, Closeable {
validateJAAS(jaas);
validateNTPConf();
- if (checkShortName) {
- validateShortName();
- }
-
if (!nologin) {
title("Logging in");
if (keytab != null) {
@@ -449,32 +431,6 @@ public class KDiag extends Configured implements Tool, Closeable {
}
/**
- * Verify whether auth_to_local rules transform a principal name
- * <p>
- * Having a local user name "bar@foo.com" may be harmless, so it is noted at
- * info. However if what was intended is a transformation to "bar"
- * it can be difficult to debug, hence this check.
- */
- protected void validateShortName() {
- failif(principal == null, CAT_KERBEROS, "No principal defined");
-
- try {
- KerberosName kn = new KerberosName(principal);
- String result = kn.getShortName();
- if (nonSimplePattern.matcher(result).find()) {
- warn(CAT_KERBEROS, principal + " short name: " + result
- + " still contains @ or /");
- }
- } catch (IOException e) {
- throw new KerberosDiagsFailure(CAT_KERBEROS, e,
- "Failed to get short name for " + principal, e);
- } catch (IllegalArgumentException e) {
- error(CAT_KERBEROS, "KerberosName(" + principal + ") failed: %s\n%s",
- e, StringUtils.stringifyException(e));
- }
- }
-
- /**
* Get the default realm.
* <p>
* Not having a default realm may be harmless, so is noted at info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index ecf94eb..09f1eed 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -470,7 +470,6 @@ KDiag: Diagnose Kerberos Problems
[--out <file>] : Write output to a file.
[--resource <resource>] : Load an XML configuration resource.
[--secure] : Require the hadoop configuration to be secure.
- [--verifyshortname <principal>]: Verify the short name of the specific principal does not contain '@' or '/'
```
#### `--jaas`: Require a JAAS file to be defined in `java.security.auth.login.config`.
@@ -566,11 +565,6 @@ or implicitly set to "simple":
Needless to say, an application so configured cannot talk to a secure Hadoop cluster.
-#### `--verifyshortname <principal>`: validate the short name of a principal
-
-This verifies that the short name of a principal contains neither the `"@"`
-nor `"/"` characters.
-
### Example
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
index e395566..3895ae1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
@@ -165,22 +165,6 @@ public class TestKDiag extends Assert {
}
@Test
- public void testKerberosName() throws Throwable {
- kdiagFailure(ARG_KEYLEN, KEYLEN,
- ARG_VERIFYSHORTNAME,
- ARG_PRINCIPAL, "foo/foo/foo@BAR.COM");
- }
-
- @Test
- public void testShortName() throws Throwable {
- kdiag(ARG_KEYLEN, KEYLEN,
- ARG_KEYTAB, keytab.getAbsolutePath(),
- ARG_PRINCIPAL,
- ARG_VERIFYSHORTNAME,
- ARG_PRINCIPAL, "foo@EXAMPLE.COM");
- }
-
- @Test
public void testFileOutput() throws Throwable {
File f = new File("target/kdiag.txt");
kdiag(ARG_KEYLEN, KEYLEN,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 3020f9b..8af0328 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -332,15 +332,10 @@ public class TestUserGroupInformation {
UserGroupInformation.setConfiguration(conf);
testConstructorSuccess("user1", "user1");
testConstructorSuccess("user4@OTHER.REALM", "other-user4");
-
- // pass through test, no transformation
- testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM");
- testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM");
- testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
- // failures
- testConstructorFailures("user6@example.com@OTHER.REALM");
- testConstructorFailures("user7@example.com@DEFAULT.REALM");
+ // failure test
+ testConstructorFailures("user2@DEFAULT.REALM");
+ testConstructorFailures("user3/cron@DEFAULT.REALM");
+ testConstructorFailures("user5/cron@OTHER.REALM");
testConstructorFailures(null);
testConstructorFailures("");
}
@@ -354,13 +349,10 @@ public class TestUserGroupInformation {
testConstructorSuccess("user1", "user1");
testConstructorSuccess("user2@DEFAULT.REALM", "user2");
- testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
-
- // no rules applied, local name remains the same
- testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM");
- testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
+ testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
// failure test
+ testConstructorFailures("user4@OTHER.REALM");
+ testConstructorFailures("user5/cron@OTHER.REALM");
testConstructorFailures(null);
testConstructorFailures("");
}
@@ -401,9 +393,8 @@ public class TestUserGroupInformation {
} catch (IllegalArgumentException e) {
String expect = (userName == null || userName.isEmpty())
? "Null user" : "Illegal principal name "+userName;
- String expect2 = "Malformed Kerberos name: "+userName;
- assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e,
- e.toString().contains(expect) || e.toString().contains(expect2));
+ assertTrue("Did not find "+ expect + " in " + e,
+ e.toString().contains(expect));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-547. Fix secure docker and
configs. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-547. Fix secure docker and configs. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eddbe997
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eddbe997
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eddbe997
Branch: refs/heads/HDDS-4
Commit: eddbe997d86c54c21763b45de8d7f4a89a97dfa2
Parents: 87acc15
Author: Ajay Kumar <aj...@apache.com>
Authored: Mon Oct 1 11:03:27 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
hadoop-dist/src/main/compose/ozonesecure/.env | 18 +++
.../compose/ozonesecure/docker-compose.yaml | 57 +++++++
.../src/main/compose/ozonesecure/docker-config | 103 +++++++++++++
.../ozonesecure/docker-image/runner/Dockerfile | 39 +++++
.../ozonesecure/docker-image/runner/build.sh | 26 ++++
.../docker-image/runner/scripts/envtoconf.py | 115 ++++++++++++++
.../docker-image/runner/scripts/krb5.conf | 38 +++++
.../docker-image/runner/scripts/starter.sh | 100 +++++++++++++
.../runner/scripts/transformation.py | 150 +++++++++++++++++++
hadoop-hdds/common/pom.xml | 6 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 3 -
.../common/src/main/resources/ozone-default.xml | 6 +-
.../hadoop/ozone/HddsDatanodeService.java | 30 ++++
.../StorageContainerManagerHttpServer.java | 4 +-
.../src/test/compose/compose-secure/.env | 17 ---
.../compose/compose-secure/docker-compose.yaml | 66 --------
.../test/compose/compose-secure/docker-config | 99 ------------
.../apache/hadoop/ozone/om/OMConfigKeys.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 11 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 4 +-
21 files changed, 701 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ image: ahadoop/kdc:v1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ image: ahadoop/runner:latest
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: ahadoop/runner:latest
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: ahadoop/runner:latest
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+ wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+ cd "$DIR"/build || exit
+ tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+ """Simple conversion"""
+ def __init__(self, args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--destination", help="Destination directory", required=True)
+ self.args = parser.parse_args(args=args)
+ # copy the default files to file.raw in destination directory
+
+ self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+ self.output_dir = self.args.destination
+
+ self.configurables = {}
+
+ def destination_file_path(self, name, extension):
+ """destination file path"""
+ return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+ def write_env_var(self, name, extension, key, value):
+ """Write environment variables"""
+ with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+ myfile.write("{}: {}\n".format(key, value))
+
+ def process_envs(self):
+ """Process environment variables"""
+ for key in os.environ.keys():
+ pattern = re.compile("[_\\.]")
+ parts = pattern.split(key)
+ extension = None
+ name = parts[0].lower()
+ if len(parts) > 1:
+ extension = parts[1].lower()
+ config_key = key[len(name) + len(extension) + 2:].strip()
+ if extension and "!" in extension:
+ splitted = extension.split("!")
+ extension = splitted[0]
+ fmt = splitted[1]
+ config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+ else:
+ fmt = extension
+
+ if extension and extension in self.known_formats:
+ if name not in self.configurables.keys():
+ with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+ myfile.write("")
+ self.configurables[name] = (extension, fmt)
+ self.write_env_var(name, extension, config_key, os.environ[key])
+ else:
+ for configurable_name in self.configurables:
+ if key.lower().startswith(configurable_name.lower()):
+ self.write_env_var(configurable_name,
+ self.configurables[configurable_name],
+ key[len(configurable_name) + 1:],
+ os.environ[key])
+
+ def transform(self):
+ """transform"""
+ for configurable_name in self.configurables:
+ name = configurable_name
+ extension, fmt = self.configurables[name]
+
+ destination_path = self.destination_file_path(name, extension)
+
+ with open(destination_path + ".raw", "r") as myfile:
+ content = myfile.read()
+ transformer_func = getattr(transformation, "to_" + fmt)
+ content = transformer_func(content)
+ with open(destination_path, "w") as myfile:
+ myfile.write(content)
+
+ def main(self):
+ """main"""
+
+ # add the
+ self.process_envs()
+
+ # copy file.ext.raw to file.ext in the destination directory, and
+ # transform to the right format (eg. key: value ===> XML)
+ self.transform()
+
+
+def main():
+ """main"""
+ Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+ Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = SERVER
+ admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+ echo "Setting up kerberos!!"
+ KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+ ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+ echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+ if [ -n "$SLEEP_SECONDS" ]; then
+ echo "Sleeping for $(SLEEP_SECONDS) seconds"
+ sleep "$SLEEP_SECONDS"
+ fi
+
+
+ while true
+ do
+ STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+ if [ "$STATUS" -eq 200 ]; then
+ echo "Got 200, KDC service ready!!"
+ break
+ else
+ echo "Got $STATUS :( KDC service not ready yet..."
+ fi
+ sleep 5
+ done
+
+ HOST_NAME=$(hostname -f)
+ export HOST_NAME
+ for NAME in ${KERBEROS_KEYTABS}; do
+ echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+ wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+ klist -kt "$CONF_DIR/$NAME.keytab"
+ KERBEROS_ENABLED=true
+ done
+
+ sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+ CLUSTERID_OPTS=""
+ if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+ CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+ fi
+ if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+ fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+ fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+ /opt/hadoop/bin/ozone scm -init
+ fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+ #To make sure SCM is running in dockerized environment we will sleep
+ # Could be removed after HDFS-13203
+ echo "Waiting 15 seconds for SCM startup"
+ sleep 15
+ /opt/hadoop/bin/ozone om -createObjectStore
+ fi
+fi
+
+echo 'setup finished'
+"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+ """render yaml"""
+ result = ""
+ if isinstance(yaml_root, dict):
+ if prefix:
+ result += "\n"
+ for key in yaml_root:
+ result += "{}{}: {}".format(prefix, key, render_yaml(
+ yaml_root[key], prefix + " "))
+ elif isinstance(yaml_root, list):
+ result += "\n"
+ for item in yaml_root:
+ result += prefix + " - " + render_yaml(item, prefix + " ")
+ else:
+ result += "{}\n".format(yaml_root)
+ return result
+
+
+def to_yaml(content):
+ """transform to yaml"""
+ props = process_properties(content)
+
+ keys = props.keys()
+ yaml_props = {}
+ for key in keys:
+ parts = key.split(".")
+ node = yaml_props
+ prev_part = None
+ parent_node = {}
+ for part in parts[:-1]:
+ if part.isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ while len(node) <= int(part):
+ node.append({})
+ parent_node = node
+ node = node[int(node)]
+ else:
+ if part not in node:
+ node[part] = {}
+ parent_node = node
+ node = node[part]
+ prev_part = part
+ if parts[-1].isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ node.append(props[key])
+ else:
+ node[parts[-1]] = props[key]
+
+ return render_yaml(yaml_props)
+
+
+def to_yml(content):
+ """transform to yml"""
+ return to_yaml(content)
+
+
+def to_properties(content):
+ """transform to properties"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props.items():
+ result += "{}: {}\n".format(key, val)
+ return result
+
+
+def to_env(content):
+ """transform to environment variables"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_sh(content):
+ """transform to shell"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}=\"{}\"\n".format(key, val)
+ return result
+
+
+def to_cfg(content):
+ """transform to config"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_conf(content):
+ """transform to configuration"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}={}\n".format(key, val)
+ return result
+
+
+def to_xml(content):
+ """transform to xml"""
+ result = "<configuration>\n"
+ props = process_properties(content)
+ for key in props:
+ result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+ format(key, props[key])
+ result += "</configuration>"
+ return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+ """
+ Read the file passed as parameter as a properties file.
+ """
+ props = {}
+ for line in content.split("\n"):
+ sline = line.strip()
+ if sline and not sline.startswith(comment_char):
+ key_value = sline.split(sep)
+ key = key_value[0].strip()
+ value = sep.join(key_value[1:]).strip().strip('"')
+ props[key] = value
+
+ return props
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 15ae307..30dcc58 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -118,6 +118,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>bcprov-jdk15on</artifactId>
<version>1.54</version>
</dependency>
+
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>1.54</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index cb6672f..abaa0f8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -314,11 +314,11 @@ public final class ScmConfigKeys {
"10m";
public static final String
- HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
- "hdds.scm.web.authentication.kerberos.principal";
+ HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
+ "hdds.scm.http.kerberos.principal";
public static final String
- HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
- "hdds.scm.web.authentication.kerberos.keytab";
+ HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.http.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index b47113b..ac97422 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -311,9 +311,6 @@ public final class OzoneConfigKeys {
DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT;
- public static final String OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.web.authentication.kerberos.principal";
-
public static final String HDDS_DATANODE_PLUGINS_KEY =
"hdds.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7d91dc8..5fb594b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1430,11 +1430,11 @@
</property>
<property>
- <name>hdds.scm.web.authentication.kerberos.principal</name>
+ <name>hdds.scm.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
</property>
<property>
- <name>hdds.scm.web.authentication.kerberos.keytab</name>
+ <name>hdds.scm.http.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
@@ -1446,7 +1446,7 @@
</description>
</property>
<property>
- <name>ozone.om.http.kerberos.keytab.file</name>
+ <name>ozone.om.http.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
OzoneManager http server kerberos keytab.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 348196c..8865368 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,11 +25,15 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
@@ -98,12 +102,38 @@ public class HddsDatanodeService implements ServicePlugin {
datanodeDetails = initializeDatanodeDetails();
datanodeDetails.setHostName(hostname);
datanodeDetails.setIpAddress(ip);
+ LOG.info("HddsDatanodeService host:{} ip:{}", hostname, ip);
+ // Authenticate Hdds Datanode service if security is enabled
+ if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY,
+ true)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ UserGroupInformation.AuthenticationMethod.KERBEROS)) {
+ LOG.debug("Ozone security is enabled. Attempting login for Hdds " +
+ "Datanode user. "
+ + "Principal: {},keytab: {}", conf.get(
+ DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY),
+ conf.get(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY));
+
+ UserGroupInformation.setConfiguration(conf);
+
+ SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY,
+ DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hostname);
+ } else {
+ throw new AuthenticationException(SecurityUtil.
+ getAuthenticationMethod(conf) + " authentication method not " +
+ "supported. Datanode user" + " login " + "failed.");
+ }
+ LOG.info("Hdds Datanode login successful.");
+ }
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
startPlugins();
// Starting HDDS Daemons
datanodeStateMachine.startDaemon();
} catch (IOException e) {
throw new RuntimeException("Can't start the HDDS datanode plugin", e);
+ } catch (AuthenticationException ex) {
+ throw new RuntimeException("Fail to authentication when starting" +
+ " HDDS datanode plugin", ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 41dd89a..dce2a45 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
deleted file mode 100644
index 199d52c..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
deleted file mode 100644
index db211bc..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- ozone.kdc:
- image: ahadoop/kdc:v1
- namenode:
- image: ahadoop/ozone:v1
- hostname: namenode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9000:9000
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
- datanode:
- image: ahadoop/ozone:v1
- hostname: datanode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9874
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","datanode"]
- om:
- image: ahadoop/ozone:v1
- hostname: om
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","ksm"]
- scm:
- image: ahadoop/ozone:v1
- hostname: scm
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
deleted file mode 100644
index cdf72ce..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.ksm.address=om
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_hdds.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=ozone.kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=10
-KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 0119eb5..303ebd9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -91,8 +91,8 @@ public final class OMConfigKeys {
+ "kerberos.keytab.file";
public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ ".kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ public static final String OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE =
"ozone.om.http.kerberos.keytab.file";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
= "ozone.om.http.kerberos.principal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 14c6800..5791de9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -117,9 +117,9 @@ public final class TestSecureOzoneCluster {
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
conf.get(ScmConfigKeys
- .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
createPrincipal(omKeyTab,
conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
@@ -153,12 +153,12 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm);
- conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY,
"HTTP_OM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab");
@@ -167,7 +167,8 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(
+ ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eddbe997/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index be12fa8..8f7f058 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -64,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
+ return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
}
@Override protected String getSpnegoPrincipal() {
- return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-778. Add an interface for CA
and Clients for Certificate operations Contributed by Anu Engineer.
Posted by xy...@apache.org.
HDDS-778. Add an interface for CA and Clients for Certificate operations
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4770e9de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4770e9de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4770e9de
Branch: refs/heads/HDDS-4
Commit: 4770e9dea8199753962c6517bff96fd39fbfd826
Parents: 8bbc95e
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Nov 8 09:54:27 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../authority/CertificateServer.java | 99 ++++++++++++
.../certificate/authority/package-info.java | 22 +++
.../certificate/client/CertificateClient.java | 159 +++++++++++++++++++
.../x509/certificate/client/package-info.java | 22 +++
4 files changed, 302 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
new file mode 100644
index 0000000..9332e5b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+
+import java.security.cert.X509Certificate;
+import java.util.concurrent.Future;
+
+/**
+ * Interface for Certificate Authority. This can be extended to talk to external
+ * CAs later or HSMs later.
+ */
+public interface CertificateServer {
+ /**
+ * Initialize the Certificate Authority.
+ *
+ * @param securityConfig - Security Configuration.
+ * @param type - The Type of CertificateServer we are creating, we make this
+ * explicit so that when we read code it is visible to the users.
+ * @throws SCMSecurityException - Throws if the init fails.
+ */
+ void init(SecurityConfig securityConfig, CAType type)
+ throws SCMSecurityException;
+
+ /**
+ * Returns the CA Certificate for this CA.
+ *
+ * @return X509CertificateHolder - Certificate for this CA.
+ * @throws SCMSecurityException -- usually thrown if this CA is not
+ * initialized.
+ */
+ X509CertificateHolder getCACertificate()
+ throws SCMSecurityException;
+
+ /**
+ * Request a Certificate based on Certificate Signing Request.
+ *
+ * @param csr - Certificate Signing Request.
+ * @return A future that will have this certificate when this request is
+ * approved.
+ * @throws SCMSecurityException - on Error.
+ */
+ Future<X509CertificateHolder> requestCertificate(CertificateSignRequest csr,
+ CertificateApprover approver) throws SCMSecurityException;
+
+ /**
+ * Revokes a Certificate issued by this CertificateServer.
+ *
+ * @param certificate - Certificate to revoke
+ * @param approver - Approval process to follow.
+ * @return Future that tells us what happened.
+ * @throws SCMSecurityException - on Error.
+ */
+ Future<Boolean> revokeCertificate(X509Certificate certificate,
+ CertificateApprover approver) throws SCMSecurityException;
+
+ /**
+ * TODO : CRL, OCSP etc. Later. This is the start of a CertificateServer
+ * framework.
+ */
+
+ /**
+ * Approval Types for a certificate request.
+ */
+ enum CertificateApprover {
+ KERBEROS_TRUSTED, /* The Request came from a DN using Kerberos Identity*/
+ MANUAL, /* Wait for a Human being to approve this certificate */
+ TESTING_AUTOMATIC /* For testing purpose, Automatic Approval. */
+ }
+
+ /**
+ * Make it explicit what type of CertificateServer we are creating here.
+ */
+ enum CAType {
+ SELF_SIGNED_CA,
+ INTERMEDIARY_CA
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
new file mode 100644
index 0000000..af53904
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to Certificate Life Cycle or Certificate Authority Server.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
new file mode 100644
index 0000000..1b6f576
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.client;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+
+import java.io.InputStream;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.cert.CertStore;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
+/**
+ * Certificate client provides and interface to certificate operations that
+ * needs to be performed by all clients in the Ozone eco-system.
+ */
+public interface CertificateClient {
+
+ /**
+ * Returns the private key of the specified component if it exists on the
+ * local system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return private key or Null if there is no data.
+ */
+ PrivateKey getPrivateKey(String component);
+
+ /**
+ * Returns the public key of the specified component if it exists on the local
+ * system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return public key or Null if there is no data.
+ */
+ PublicKey getPublicKey(String component);
+
+ /**
+ * Returns the certificate of the specified component if it exists on the
+ * local system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return certificate or Null if there is no data.
+ */
+ X509Certificate getCertificate(String component);
+
+ /**
+ * Verifies if this certificate is part of a trusted chain.
+ *
+ * @return true if it trusted, false otherwise.
+ */
+ boolean verifyCertificate(X509Certificate certificate);
+
+ /**
+ * Creates digital signature over the data stream using the components private
+ * key.
+ *
+ * @param stream - Data stream to sign.
+ * @return byte array - containing the signature.
+ */
+ byte[] signDataStream(InputStream stream, String component)
+ throws CertificateException;
+
+ /**
+ * Verifies a digital Signature, given the signature and the certificate of
+ * the signer.
+ * @param stream - Data Stream.
+ * @param signature - Byte Array containing the signature.
+ * @param cert - Certificate of the Signer.
+ * @return true if verified, false if not.
+ */
+ boolean verifySignature(InputStream stream, byte[] signature,
+ X509Certificate cert);
+
+ /**
+ * Returns a CSR builder that can be used to creates a Certificate sigining
+ * request.
+ *
+ * @return CertificateSignRequest.Builder
+ */
+ CertificateSignRequest.Builder getCSRBuilder();
+
+ /**
+ * Get the certificate of well-known entity from SCM.
+ *
+ * @param query - String Query, please see the implementation for the
+ * discussion on the query formats.
+ * @return X509Certificate or null if not found.
+ */
+ X509Certificate queryCertificate(String query);
+
+ /**
+ * Stores the private key of a specified component.
+ *
+ * @param key - private key
+ * @param component - name of the component.
+ * @throws CertificateException
+ */
+ void storePrivateKey(PrivateKey key, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the public key of a specified component.
+ *
+ * @param key - public key
+ * @throws CertificateException
+ */
+ void storePublicKey(PublicKey key, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the Certificate of a specific component.
+ *
+ * @param certificate - X509 Certificate
+ * @param component - Name of the component.
+ * @throws CertificateException
+ */
+ void storeCertificate(X509Certificate certificate, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the trusted chain of certificates for a specific component.
+ *
+ * @param certStore - Cert Store.
+ * @param component - Trust Chain.
+ * @throws CertificateException
+ */
+ void storeTrustChain(CertStore certStore,
+ String component) throws CertificateException;
+
+ /**
+ * Stores the trusted chain of certificates for a specific component.
+ *
+ * @param certificates - List of Certificates.
+ * @param component - String component.
+ * @throws CertificateException
+ */
+ void storeTrustChain(List<X509Certificate> certificates,
+ String component) throws CertificateException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4770e9de/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
new file mode 100644
index 0000000..dea609b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to creating and using certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.client;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager
Delegation Token support. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-8. Add OzoneManager Delegation Token support. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59767be1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59767be1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59767be1
Branch: refs/heads/HDDS-4
Commit: 59767be13a37557b6d51cd5356c58ebdfad63896
Parents: 9c0baf0
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 15 12:18:19 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 33 +
.../ozone/client/protocol/ClientProtocol.java | 32 +
.../hadoop/ozone/client/rest/RestClient.java | 41 ++
.../hadoop/ozone/client/rpc/RpcClient.java | 41 ++
.../ozone/client/TestHddsClientUtils.java | 2 +-
.../java/org/apache/hadoop/ozone/OmUtils.java | 17 +-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 13 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 2 +-
.../protocol/OzoneManagerSecurityProtocol.java | 67 +++
...neManagerProtocolClientSideTranslatorPB.java | 77 +++
.../om/protocolPB/OzoneManagerProtocolPB.java | 3 +
.../hadoop/ozone/protocolPB/OMPBHelper.java | 38 ++
.../security/OzoneDelegationTokenSelector.java | 3 +-
.../ozone/security/OzoneSecretManager.java | 598 +++++++++++++++++++
.../hadoop/ozone/security/OzoneSecretStore.java | 250 ++++++++
.../ozone/security/OzoneSecurityException.java | 104 ++++
.../src/main/proto/OzoneManagerProtocol.proto | 19 +
.../ozone/security/TestOzoneSecretManager.java | 216 +++++++
.../hadoop/ozone/TestSecureOzoneCluster.java | 309 ++++++++--
.../apache/hadoop/ozone/om/OzoneManager.java | 300 ++++++++--
...neManagerProtocolServerSideTranslatorPB.java | 59 ++
23 files changed, 2150 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index ac97422..742fe3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -332,6 +332,10 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
+ public static final String OZONE_MAX_KEY_LEN =
+ "ozone.max.key.len";
+ public static final int OZONE_MAX_KEY_LEN_DEFAULT = 1024 * 1024;
+
/**
* Config properties to set client side checksum properties.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 096baee..e7c354c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -99,6 +99,7 @@ public final class OzoneConsts {
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String OM_DB_NAME = "om.db";
+ public static final String OZONE_MANAGER_TOKEN_DB_NAME = "om-token.db";
public static final String STORAGE_DIR_CHUNKS = "chunks";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5fb594b..766f31d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -984,6 +984,15 @@
every principal specified in the keytab file.
</description>
</property>
+ <property>
+ <name>ozone.max.key.len</name>
+ <value>1048576</value>
+ <tag>OZONE, SECURITY</tag>
+ <description>
+ Maximum length of private key in Ozone. Used in Ozone delegation and
+ block tokens.
+ </description>
+ </property>
<!--Client Settings-->
<property>
@@ -1492,4 +1501,28 @@
Name of file which stores public key generated for SCM CA.
</description>
</property>
+ <property>
+ <name>ozone.manager.delegation.remover.scan.interval</name>
+ <value>3600000</value>
+ <description>
+ Time interval after which ozone secret manger scans for expired
+ delegation token.
+ </description>
+ </property>
+ <property>
+ <name>ozone.manager.delegation.token.renew-interval</name>
+ <value>1d</value>
+ <description>
+ Default time interval after which ozone delegation token will
+ require renewal before any further use.
+ </description>
+ </property>
+ <property>
+ <name>ozone.manager.delegation.token.max-lifetime</name>
+ <value>7d</value>
+ <description>
+ Default max time interval after which ozone delegation token will
+ not be renewed.
+ </description>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 17634f2..2fe302e 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.protocol;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -30,7 +31,9 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
/**
* An implementer of this interface is capable of connecting to Ozone Cluster
@@ -388,4 +391,33 @@ public interface ClientProtocol {
*/
void close() throws IOException;
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException;
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 3fd0927..6e2841d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -44,10 +45,12 @@ import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
@@ -667,6 +670,44 @@ public class RestClient implements ClientProtocol {
}
}
+ /**
+ * Get a valid Delegation Token. Not supported for RestClient.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
+ /**
+ * Renew an existing delegation token. Not supported for RestClient.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
+ /**
+ * Cancel an existing delegation token. Not supported for RestClient.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
@Override
public OzoneInputStream getKey(
String volumeName, String bucketName, String keyName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 1b70f3b..3575f01 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -67,7 +67,10 @@ import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -404,6 +407,44 @@ public class RpcClient implements ClientProtocol {
ozoneManagerClient.setBucketProperty(builder.build());
}
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ return ozoneManagerClient.getDelegationToken(renewer);
+ }
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ return ozoneManagerClient.renewDelegationToken(token);
+ }
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ ozoneManagerClient.cancelDelegationToken(token);
+ }
+
@Override
public void setBucketVersioning(
String volumeName, String bucketName, Boolean versioning)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
index 9e60e4e..ff4aeb3 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
@@ -84,7 +84,7 @@ public class TestHddsClientUtils {
}
@Test
- public void testGetOmAddress() {
+ public void testgetOmSocketAddress() {
final Configuration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 9280648..e7ed84f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -54,14 +54,21 @@ public final class OmUtils {
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
- public static InetSocketAddress getOmAddress(
- Configuration conf) {
+ public static InetSocketAddress getOmAddress(Configuration conf) {
+ return NetUtils.createSocketAddr(getOmRpcAddress(conf));
+ }
+
+ /**
+ * Retrieve the socket address that is used by OM.
+ * @param conf
+ * @return Target InetSocketAddress for the SCM service endpoint.
+ */
+ public static String getOmRpcAddress(Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
- return NetUtils.createSocketAddr(
- host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
- getOmRpcPort(conf));
+ return host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
+ getOmRpcPort(conf);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 303ebd9..de24184 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -95,4 +95,17 @@ public final class OMConfigKeys {
"ozone.om.http.kerberos.keytab.file";
public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
= "ozone.om.http.kerberos.principal";
+ // Delegation token related keys
+ public static final String DELEGATION_REMOVER_SCAN_INTERVAL_KEY =
+ "ozone.manager.delegation.remover.scan.interval";
+ public static final long DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT =
+ 60*60*1000;
+ public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
+ "ozone.manager.delegation.token.renew-interval";
+ public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
+ 24*60*60*1000; // 1 day = 86400000 ms
+ public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY =
+ "ozone.manager.delegation.token.max-lifetime";
+ public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
+ 7*24*60*60*1000; // 7 days
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 233666a..06933d2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.KerberosInfo;
*/
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
-public interface OzoneManagerProtocol {
+public interface OzoneManagerProtocol extends OzoneManagerSecurityProtocol {
/**
* Creates a volume.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
new file mode 100644
index 0000000..15873d0
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * Security protocol for a secure OzoneManager.
+ */
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+public interface OzoneManagerSecurityProtocol {
+
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Idempotent
+ Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException;
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Idempotent
+ long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Idempotent
+ void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 6c8c932..85ae4a8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
@@ -139,6 +140,13 @@ import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.stream.Collectors;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.token.Token;
/**
* The client side implementation of OzoneManagerProtocol.
@@ -884,4 +892,73 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
public Object getUnderlyingProxyObject() {
return null;
}
+
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ GetDelegationTokenRequestProto req = GetDelegationTokenRequestProto
+ .newBuilder()
+ .setRenewer(renewer == null ? "" : renewer.toString())
+ .build();
+ try {
+ GetDelegationTokenResponseProto resp =
+ rpcProxy.getDelegationToken(NULL_RPC_CONTROLLER, req);
+ return resp.hasToken() ?
+ OMPBHelper.convertToDelegationToken(resp.getToken()) : null;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ } catch (Exception e){
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ RenewDelegationTokenRequestProto req =
+ RenewDelegationTokenRequestProto.newBuilder().
+ setToken(OMPBHelper.convertToTokenProto(token)).
+ build();
+ try {
+ return rpcProxy.renewDelegationToken(NULL_RPC_CONTROLLER, req)
+ .getNewExpiryTime();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ CancelDelegationTokenRequestProto req = CancelDelegationTokenRequestProto
+ .newBuilder()
+ .setToken(OMPBHelper.convertToTokenProto(token))
+ .build();
+ try {
+ rpcProxy.cancelDelegationToken(NULL_RPC_CONTROLLER, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index 27e8f22..175527b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService;
import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
/**
* Protocol used to communicate with OM.
@@ -32,6 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
protocolVersion = 1)
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+@TokenInfo(OzoneDelegationTokenSelector.class)
@InterfaceAudience.Private
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
index d57d32e..df069ce 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.protocolPB;
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
@@ -24,6 +26,9 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.security.token.Token;
/**
* Utilities for converting protobuf classes.
@@ -110,4 +115,37 @@ public final class OMPBHelper {
return new OzoneAcl(aclType, aclInfo.getName(), aclRights);
}
+
+ /**
+ * Converts Ozone delegation token to @{@link TokenProto}.
+ * @return tokenProto
+ */
+ public static TokenProto convertToTokenProto(Token<?> tok) {
+ if(tok == null){
+ throw new IllegalArgumentException("Invalid argument: token is null");
+ }
+
+ return TokenProto.newBuilder().
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
+ setKind(tok.getKind().toString()).
+ setService(tok.getService().toString()).build();
+ }
+
+ public static ByteString getByteString(byte[] bytes) {
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
+ }
+
+ /**
+ * Converts @{@link TokenProto} to Ozone delegation token.
+ *
+ * @return Ozone
+ */
+ public static Token<OzoneTokenIdentifier> convertToDelegationToken(
+ TokenProto tokenProto) {
+ return new Token<>(tokenProto.getIdentifier()
+ .toByteArray(), tokenProto.getPassword().toByteArray(), new Text(
+ tokenProto.getKind()), new Text(tokenProto.getService()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
index 0d63c4e..8e35f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.security;
import java.util.Collection;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -32,7 +31,7 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
public class OzoneDelegationTokenSelector
- extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+ extends AbstractDelegationTokenSelector<OzoneTokenIdentifier> {
public OzoneDelegationTokenSelector() {
super(OzoneTokenIdentifier.KIND_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
new file mode 100644
index 0000000..0c84404
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
@@ -0,0 +1,598 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SecretManager for Ozone Master. Responsible for signing identifiers with
+ * private key,
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretManager<T extends OzoneTokenIdentifier>
+ extends SecretManager<T> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneSecretManager.class);
+ /**
+ * The name of the Private/Public Key based hashing algorithm.
+ */
+ private static final String DEFAULT_SIGNATURE_ALGORITHM = "SHA256withRSA";
+ private final long tokenMaxLifetime;
+ private final long tokenRenewInterval;
+ private final long tokenRemoverScanInterval;
+ private final Text service;
+ private final Map<Integer, OzoneSecretKey> allKeys;
+ private final Map<T, TokenInfo> currentTokens;
+ private final OzoneSecretStore store;
+ private Thread tokenRemoverThread;
+ private volatile boolean running;
+ private AtomicInteger tokenSequenceNumber;
+ private OzoneSecretKey currentKey;
+ private AtomicInteger currentKeyId;
+ /**
+ * If the delegation token update thread holds this lock, it will not get
+ * interrupted.
+ */
+ private Object noInterruptsLock = new Object();
+ private int maxKeyLength;
+
+ /**
+ * Create a secret manager.
+ *
+ * @param conf configuration.
+ * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
+ * milliseconds
+ * @param tokenRenewInterval how often the tokens must be renewed in
+ * milliseconds
+ * @param dtRemoverScanInterval how often the tokens are scanned for expired
+ * tokens in milliseconds
+ */
+ public OzoneSecretManager(OzoneConfiguration conf, long tokenMaxLifetime,
+ long tokenRenewInterval, long dtRemoverScanInterval, Text service)
+ throws IOException {
+ this.tokenMaxLifetime = tokenMaxLifetime;
+ this.tokenRenewInterval = tokenRenewInterval;
+ this.tokenRemoverScanInterval = dtRemoverScanInterval;
+
+ currentTokens = new ConcurrentHashMap();
+ allKeys = new ConcurrentHashMap<>();
+ currentKeyId = new AtomicInteger();
+ tokenSequenceNumber = new AtomicInteger();
+ this.store = new OzoneSecretStore(conf);
+ loadTokenSecretState(store.loadState());
+ this.service = service;
+ this.maxKeyLength = conf.getInt(OzoneConfigKeys.OZONE_MAX_KEY_LEN,
+ OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
+ }
+
+ @Override
+ public T createIdentifier() {
+ return (T) T.newInstance();
+ }
+
+ /**
+ * Create new Identifier with given,owner,renwer and realUser.
+ *
+ * @return T
+ */
+ public T createIdentifier(Text owner, Text renewer, Text realUser) {
+ return (T) T.newInstance(owner, renewer, realUser);
+ }
+
+ /**
+ * Returns {@link Token} for given identifier.
+ *
+ * @param owner
+ * @param renewer
+ * @param realUser
+ * @return Token
+ * @throws IOException to allow future exceptions to be added without breaking
+ * compatibility
+ */
+ public Token<T> createToken(Text owner, Text renewer, Text realUser)
+ throws IOException {
+ T identifier = createIdentifier(owner, renewer, realUser);
+ updateIdentifierDetails(identifier);
+
+ byte[] password = createPassword(identifier.getBytes(),
+ currentKey.getPrivateKey());
+ addToTokenStore(identifier, password);
+ Token<T> token = new Token<>(identifier.getBytes(), password,
+ identifier.getKind(), service);
+ if (LOG.isTraceEnabled()) {
+ long expiryTime = identifier.getIssueDate() + tokenRenewInterval;
+ String tokenId = identifier.toStringStable();
+ LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+ expiryTime, tokenId);
+ }
+
+ return token;
+ }
+
+ /**
+ * Stores given identifier in token store.
+ *
+ * @param identifier
+ * @param password
+ * @throws IOException
+ */
+ private void addToTokenStore(T identifier, byte[] password)
+ throws IOException {
+ TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
+ + tokenRenewInterval, password, identifier.getTrackingId());
+ currentTokens.put(identifier, tokenInfo);
+ store.storeToken(identifier, tokenInfo.getRenewDate());
+ }
+
+ /**
+ * Updates issue date, master key id and sequence number for identifier.
+ *
+ * @param identifier the identifier to validate
+ */
+ private void updateIdentifierDetails(T identifier) {
+ int sequenceNum;
+ long now = Time.monotonicNow();
+ sequenceNum = incrementDelegationTokenSeqNum();
+ identifier.setIssueDate(now);
+ identifier.setMasterKeyId(currentKey.getKeyId());
+ identifier.setSequenceNumber(sequenceNum);
+ identifier.setMaxDate(Time.monotonicNow() + tokenMaxLifetime);
+ }
+
+ /**
+ * Compute HMAC of the identifier using the private key and return the output
+ * as password.
+ *
+ * @param identifier
+ * @param privateKey
+ * @return byte[] signed byte array
+ */
+ public byte[] createPassword(byte[] identifier, PrivateKey privateKey)
+ throws OzoneSecurityException {
+ try {
+ Signature rsaSignature = Signature.getInstance(
+ DEFAULT_SIGNATURE_ALGORITHM);
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(identifier);
+ return rsaSignature.sign();
+ } catch (InvalidKeyException | NoSuchAlgorithmException |
+ SignatureException ex) {
+ throw new OzoneSecurityException("Error while creating HMAC hash for " +
+ "token.", ex, OzoneSecurityException.ResultCodes
+ .SECRET_MANAGER_HMAC_ERROR);
+ }
+ }
+
+ @Override
+ public byte[] createPassword(T identifier) {
+ LOG.debug("Creating password for identifier: {}, currentKey: {}",
+ formatTokenId(identifier), currentKey.getKeyId());
+ byte[] password = null;
+ try {
+ password = createPassword(identifier.getBytes(),
+ currentKey.getPrivateKey());
+ } catch (IOException ioe) {
+ LOG.error("Could not store token {}!!", formatTokenId(identifier),
+ ioe);
+ }
+ return password;
+ }
+
+ @Override
+ public byte[] retrievePassword(T identifier) throws InvalidToken {
+ return checkToken(identifier).getPassword();
+ }
+
+ /**
+ * Renew a delegation token.
+ *
+ * @param token the token to renew
+ * @param renewer the full principal name of the user doing the renewal
+ * @return the new expiration time
+ * @throws InvalidToken if the token is invalid
+ * @throws AccessControlException if the user can't renew token
+ */
+ public synchronized long renewToken(Token<T> token, String renewer)
+ throws IOException {
+ ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ T id = (T) T.readProtoBuf(in);
+ LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
+ formatTokenId(id), currentTokens.size());
+
+ long now = Time.monotonicNow();
+ if (id.getMaxDate() < now) {
+ throw new InvalidToken(renewer + " tried to renew an expired token "
+ + formatTokenId(id) + " max expiration date: "
+ + Time.formatTime(id.getMaxDate())
+ + " currentTime: " + Time.formatTime(now));
+ }
+ checkToken(id);
+ if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
+ throw new AccessControlException(renewer +
+ " tried to renew a token " + formatTokenId(id)
+ + " without a renewer");
+ }
+ if (!id.getRenewer().toString().equals(renewer)) {
+ throw new AccessControlException(renewer
+ + " tries to renew a token " + formatTokenId(id)
+ + " with non-matching renewer " + id.getRenewer());
+ }
+ OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
+ if (key == null) {
+ throw new InvalidToken("Unable to find master key for keyId="
+ + id.getMasterKeyId()
+ + " from cache. Failed to renew an unexpired token "
+ + formatTokenId(id) + " with sequenceNumber="
+ + id.getSequenceNumber());
+ }
+ byte[] password = createPassword(token.getIdentifier(),
+ key.getPrivateKey());
+
+ long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
+ try {
+ addToTokenStore(id, password);
+ } catch (IOException e) {
+ LOG.error("Unable to update token " + id.getSequenceNumber(), e);
+ }
+ return renewTime;
+ }
+
+ /**
+ * Cancel a token by removing it from store and cache.
+ *
+ * @return Identifier of the canceled token
+ * @throws InvalidToken for invalid token
+ * @throws AccessControlException if the user isn't allowed to cancel
+ */
+ public T cancelToken(Token<T> token, String canceller) throws IOException {
+ T id = (T) T.readProtoBuf(token.getIdentifier());
+ LOG.debug("Token cancellation requested for identifier: {}",
+ formatTokenId(id));
+
+ if (id.getUser() == null) {
+ throw new InvalidToken("Token with no owner " + formatTokenId(id));
+ }
+ String owner = id.getUser().getUserName();
+ Text renewer = id.getRenewer();
+ HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
+ String cancelerShortName = cancelerKrbName.getShortName();
+ if (!canceller.equals(owner)
+ && (renewer == null || renewer.toString().isEmpty()
+ || !cancelerShortName
+ .equals(renewer.toString()))) {
+ throw new AccessControlException(canceller
+ + " is not authorized to cancel the token " + formatTokenId(id));
+ }
+ try {
+ store.removeToken(id);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
+ }
+ TokenInfo info = currentTokens.remove(id);
+ if (info == null) {
+ throw new InvalidToken("Token not found " + formatTokenId(id));
+ }
+ return id;
+ }
+
+ public int getCurrentKeyId() {
+ return currentKeyId.get();
+ }
+
+ public void setCurrentKeyId(int keyId) {
+ currentKeyId.set(keyId);
+ }
+
+ public int incrementCurrentKeyId() {
+ return currentKeyId.incrementAndGet();
+ }
+
+ public int getDelegationTokenSeqNum() {
+ return tokenSequenceNumber.get();
+ }
+
+ public void setDelegationTokenSeqNum(int seqNum) {
+ tokenSequenceNumber.set(seqNum);
+ }
+
+ public int incrementDelegationTokenSeqNum() {
+ return tokenSequenceNumber.incrementAndGet();
+ }
+
+ /**
+ * Validates if given token is valid.
+ *
+ * @param identifier
+ * @param password
+ */
+ private boolean validateToken(T identifier, byte[] password) {
+ try {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(currentKey.getPublicKey());
+ rsaSignature.update(identifier.getBytes());
+ return rsaSignature.verify(password);
+ } catch (NoSuchAlgorithmException | SignatureException |
+ InvalidKeyException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Checks if TokenInfo for the given identifier exists in database and if the
+ * token is expired.
+ */
+ public TokenInfo checkToken(T identifier) throws InvalidToken {
+ TokenInfo info = currentTokens.get(identifier);
+ if (info == null) {
+ throw new InvalidToken("token " + formatTokenId(identifier)
+ + " can't be found in cache");
+ }
+ long now = Time.monotonicNow();
+ if (info.getRenewDate() < now) {
+ throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+ "expired, current time: " + Time.formatTime(now) +
+ " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+ }
+ if (!validateToken(identifier, info.getPassword())) {
+ throw new InvalidToken("Tampared/Inavalid token.");
+ }
+ return info;
+ }
+
+ // TODO: handle roll private key/certificate
+ private synchronized void removeExpiredKeys() {
+ long now = Time.monotonicNow();
+ for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+ .iterator(); it.hasNext();) {
+ Map.Entry<Integer, OzoneSecretKey> e = it.next();
+ OzoneSecretKey key = e.getValue();
+ if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
+ if (!key.equals(currentKey)) {
+ it.remove();
+ try {
+ store.removeTokenMasterKey(key);
+ } catch (IOException ex) {
+ LOG.error("Unable to remove master key " + key.getKeyId(), ex);
+ }
+ }
+ }
+ }
+ }
+
+ private void loadTokenSecretState(OzoneManagerSecretState<T> state)
+ throws IOException {
+ LOG.info("Loading token state into token manager.");
+ for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
+ allKeys.putIfAbsent(key.getKeyId(), key);
+ }
+ for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
+ addPersistedDelegationToken(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private String formatTokenId(T id) {
+ return "(" + id + ")";
+ }
+
+ private void addPersistedDelegationToken(
+ T identifier, long renewDate)
+ throws IOException {
+ if (running) {
+ // a safety check
+ throw new IOException(
+ "Can't add persisted delegation token to a running SecretManager.");
+ }
+ int keyId = identifier.getMasterKeyId();
+ OzoneSecretKey dKey = allKeys.get(keyId);
+ if (dKey == null) {
+ LOG.warn("No KEY found for persisted identifier "
+ + formatTokenId(identifier));
+ return;
+ }
+
+ PrivateKey privateKey = dKey.getPrivateKey();
+ byte[] password = createPassword(identifier.getBytes(), privateKey);
+ if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
+ setDelegationTokenSeqNum(identifier.getSequenceNumber());
+ }
+ if (currentTokens.get(identifier) == null) {
+ currentTokens.put(identifier, new TokenInfo(renewDate,
+ password, identifier.getTrackingId()));
+ } else {
+ throw new IOException("Same delegation token being added twice: "
+ + formatTokenId(identifier));
+ }
+ }
+
+ /**
+ * Should be called before this object is used.
+ */
+ public void startThreads(KeyPair keyPair) throws IOException {
+ Preconditions.checkState(!running);
+ updateCurrentKey(keyPair);
+ removeExpiredKeys();
+ synchronized (this) {
+ running = true;
+ tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
+ tokenRemoverThread.start();
+ }
+ }
+
+ public void stopThreads() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping expired delegation token remover thread");
+ }
+ running = false;
+
+ if (tokenRemoverThread != null) {
+ synchronized (noInterruptsLock) {
+ tokenRemoverThread.interrupt();
+ }
+ try {
+ tokenRemoverThread.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(
+ "Unable to join on token removal thread", e);
+ }
+ }
+ }
+
+ /**
+ * Stops the OzoneSecretManager.
+ *
+ * @throws IOException
+ */
+ public void stop() throws IOException {
+ stopThreads();
+ if (this.store != null) {
+ this.store.close();
+ }
+ }
+
+ /**
+ * Update the current master key. This is called once by startThreads before
+ * tokenRemoverThread is created,
+ */
+ private void updateCurrentKey(KeyPair keyPair) throws IOException {
+ LOG.info("Updating the current master key for generating tokens");
+
+ // TODO: fix me based on the certificate expire time to set the key
+ // expire time.
+ int newCurrentId = incrementCurrentKeyId();
+ OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
+ keyPair, maxKeyLength);
+
+ store.storeTokenMasterKey(newKey);
+ if (!allKeys.containsKey(newKey.getKeyId())) {
+ allKeys.put(newKey.getKeyId(), newKey);
+ }
+
+ synchronized (this) {
+ currentKey = newKey;
+ }
+ }
+
+ /**
+ * Remove expired delegation tokens from cache and persisted store.
+ */
+ private void removeExpiredToken() throws IOException {
+ long now = Time.monotonicNow();
+ synchronized (this) {
+ Iterator<Map.Entry<T,
+ TokenInfo>> i = currentTokens.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry<T,
+ TokenInfo> entry = i.next();
+ long renewDate = entry.getValue().getRenewDate();
+ if (renewDate < now) {
+ i.remove();
+ store.removeToken(entry.getKey());
+ }
+ }
+ }
+ }
+
+ /**
+ * Is Secret Manager running.
+ *
+ * @return true if secret mgr is running
+ */
+ public synchronized boolean isRunning() {
+ return running;
+ }
+
+ /**
+ * Returns expiry time of a token given its identifier.
+ *
+ * @param dtId DelegationTokenIdentifier of a token
+ * @return Expiry time of the token
+ * @throws IOException
+ */
+ public long getTokenExpiryTime(T dtId)
+ throws IOException {
+ TokenInfo info = currentTokens.get(dtId);
+ if (info != null) {
+ return info.getRenewDate();
+ } else {
+ throw new IOException("No delegation token found for this identifier");
+ }
+ }
+
+ private class ExpiredTokenRemover extends Thread {
+ private long lastTokenCacheCleanup;
+
+ @Override
+ public void run() {
+ LOG.info("Starting expired delegation token remover thread, "
+ + "tokenRemoverScanInterval=" + tokenRemoverScanInterval
+ / (60 * 1000) + " min(s)");
+ try {
+ while (running) {
+ long now = Time.monotonicNow();
+ if (lastTokenCacheCleanup + tokenRemoverScanInterval
+ < now) {
+ removeExpiredToken();
+ lastTokenCacheCleanup = now;
+ }
+ try {
+ Thread.sleep(Math.min(5000,
+ tokenRemoverScanInterval)); // 5 seconds
+ } catch (InterruptedException ie) {
+ LOG.error("ExpiredTokenRemover received " + ie);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("ExpiredTokenRemover thread received unexpected exception",
+ t);
+ Runtime.getRuntime().exit(-1);
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
new file mode 100644
index 0000000..6528bcf
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.utils.MetadataKeyFilters;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MANAGER_TOKEN_DB_NAME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_MB;
+
+/**
+ * SecretStore for Ozone Master.
+ */
+public class OzoneSecretStore<T extends OzoneTokenIdentifier>
+ implements Closeable {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneSecretStore.class);
+ private static final String TOKEN_MASTER_KEY_KEY_PREFIX = "tokens/key_";
+ private static final String TOKEN_STATE_KEY_PREFIX = "tokens/token_";
+
+ @Override
+ public void close() throws IOException {
+ if (store != null) {
+ store.close();
+ }
+ }
+
+
+ /**
+ * Support class to maintain state of OzoneSecretStore.
+ */
+ public static class OzoneManagerSecretState<T> {
+
+ private Map<T, Long> tokenState = new HashMap<>();
+ private Set<OzoneSecretKey> tokenMasterKeyState = new HashSet<>();
+
+ public Map<T, Long> getTokenState() {
+ return tokenState;
+ }
+
+ public Set<OzoneSecretKey> ozoneManagerSecretState() {
+ return tokenMasterKeyState;
+ }
+ }
+
+ private MetadataStore store;
+
+ public OzoneSecretStore(OzoneConfiguration conf)
+ throws IOException {
+ File metaDir = getOzoneMetaDirPath(conf);
+ final int cacheSize = conf.getInt(OZONE_OM_DB_CACHE_SIZE_MB,
+ OZONE_OM_DB_CACHE_SIZE_DEFAULT);
+ File omTokenDBFile = new File(metaDir.getPath(),
+ OZONE_MANAGER_TOKEN_DB_NAME);
+ this.store = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setDbFile(omTokenDBFile)
+ .setCacheSize(cacheSize * OzoneConsts.MB)
+ .build();
+ }
+
+ public OzoneManagerSecretState loadState() throws IOException {
+ OzoneManagerSecretState state = new OzoneManagerSecretState();
+ int numKeys = loadMasterKeys(state);
+ LOG.info("Loaded " + numKeys + " token master keys");
+ int numTokens = loadTokens(state);
+ LOG.info("Loaded " + numTokens + " tokens");
+ return state;
+ }
+
+ public void storeTokenMasterKey(OzoneSecretKey key) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing master key " + key.getKeyId());
+ }
+ ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+ DataOutputStream dataStream = new DataOutputStream(memStream);
+ try {
+ key.write(dataStream);
+ dataStream.close();
+ dataStream = null;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, dataStream);
+ }
+ try {
+ byte[] dbKey = getMasterKeyDBKey(key);
+ store.put(dbKey, memStream.toByteArray());
+ } catch (IOException e) {
+ LOG.error("Unable to store master key " + key.getKeyId(), e);
+ throw e;
+ }
+ }
+
+
+ public void removeTokenMasterKey(OzoneSecretKey key)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing master key " + key.getKeyId());
+ }
+
+ byte[] dbKey = getMasterKeyDBKey(key);
+ try {
+ store.delete(dbKey);
+ } catch (IOException e) {
+ LOG.error("Unable to delete master key " + key.getKeyId(), e);
+ throw e;
+ }
+ }
+
+ public void storeToken(T tokenId, Long renewDate)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing token " + tokenId.getSequenceNumber());
+ }
+
+ ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+ DataOutputStream dataStream = new DataOutputStream(memStream);
+ try {
+ tokenId.write(dataStream);
+ dataStream.writeLong(renewDate);
+ dataStream.close();
+ dataStream = null;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, dataStream);
+ }
+
+ byte[] dbKey = getTokenDBKey(tokenId);
+ try {
+ store.put(dbKey, memStream.toByteArray());
+ } catch (IOException e) {
+ LOG.error("Unable to store token " + tokenId.toString(), e);
+ throw e;
+ }
+ }
+
+ public void updateToken(T tokenId, Long renewDate)
+ throws IOException {
+ storeToken(tokenId, renewDate);
+ }
+
+ public void removeToken(T tokenId)
+ throws IOException {
+ byte[] dbKey = getTokenDBKey(tokenId);
+ try {
+ store.delete(dbKey);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + tokenId.toString(), e);
+ throw e;
+ }
+ }
+
+ public int loadMasterKeys(OzoneManagerSecretState state) throws IOException {
+ MetadataKeyFilters.MetadataKeyFilter filter =
+ (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+ .startsWith(TOKEN_MASTER_KEY_KEY_PREFIX);
+ List<Map.Entry<byte[], byte[]>> kvs = store
+ .getRangeKVs(null, Integer.MAX_VALUE, filter);
+ kvs.forEach(entry -> {
+ try {
+ loadTokenMasterKey(state, entry.getValue());
+ } catch (IOException e) {
+ LOG.warn("Failed to load master key ",
+ DFSUtil.bytes2String(entry.getKey()), e);
+ }
+ });
+ return kvs.size();
+ }
+
+ private void loadTokenMasterKey(OzoneManagerSecretState state, byte[] data)
+ throws IOException {
+ OzoneSecretKey key = OzoneSecretKey.readProtoBuf(data);
+ state.tokenMasterKeyState.add(key);
+ }
+
+ public int loadTokens(OzoneManagerSecretState state) throws IOException {
+ MetadataKeyFilters.MetadataKeyFilter filter =
+ (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+ .startsWith(TOKEN_STATE_KEY_PREFIX);
+ List<Map.Entry<byte[], byte[]>> kvs =
+ store.getRangeKVs(null, Integer.MAX_VALUE, filter);
+ kvs.forEach(entry -> {
+ try {
+ loadToken(state, entry.getValue());
+ } catch (IOException e) {
+ LOG.warn("Failed to load token ",
+ DFSUtil.bytes2String(entry.getKey()), e);
+ }
+ });
+ return kvs.size();
+ }
+
+ private void loadToken(OzoneManagerSecretState state, byte[] data)
+ throws IOException {
+ long renewDate;
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+ T tokenId = (T) T.readProtoBuf(in);
+ try {
+ tokenId.readFields(in);
+ renewDate = in.readLong();
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, in);
+ }
+ state.tokenState.put(tokenId, renewDate);
+ }
+
+ private byte[] getMasterKeyDBKey(OzoneSecretKey masterKey) {
+ return DFSUtil.string2Bytes(
+ TOKEN_MASTER_KEY_KEY_PREFIX + masterKey.getKeyId());
+ }
+
+ private byte[] getTokenDBKey(T tokenId) {
+ return DFSUtil.string2Bytes(
+ TOKEN_STATE_KEY_PREFIX + tokenId.getSequenceNumber());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
new file mode 100644
index 0000000..66533f3
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.IOException;
+
+/**
+ * Security exceptions thrown at Ozone layer.
+ */
+public class OzoneSecurityException extends IOException {
+ private final OzoneSecurityException.ResultCodes result;
+
+ /**
+ * Constructs an {@code IOException} with {@code null}
+ * as its error detail message.
+ */
+ public OzoneSecurityException(OzoneSecurityException.ResultCodes result) {
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the
+ * {@link #getMessage()} method)
+ */
+ public OzoneSecurityException(String message,
+ OzoneSecurityException.ResultCodes result) {
+ super(message);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message
+ * and cause.
+ * <p>
+ * <p> Note that the detail message associated with {@code cause} is
+ * <i>not</i> automatically incorporated into this exception's detail
+ * message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the
+ * {@link #getMessage()} method)
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and indicates that the
+ * cause is nonexistent or unknown.)
+ * @since 1.6
+ */
+ public OzoneSecurityException(String message, Throwable cause,
+ OzoneSecurityException.ResultCodes result) {
+ super(message, cause);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified cause and a
+ * detail message of {@code (cause==null ? null : cause.toString())}
+ * (which typically contains the class and detail message of {@code cause}).
+ * This constructor is useful for IO exceptions that are little more
+ * than wrappers for other throwables.
+ *
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and indicates that the
+ * cause is nonexistent or unknown.)
+ * @since 1.6
+ */
+ public OzoneSecurityException(Throwable cause,
+ OzoneSecurityException.ResultCodes result) {
+ super(cause);
+ this.result = result;
+ }
+
+ /**
+ * Returns resultCode.
+ * @return ResultCode
+ */
+ public OzoneSecurityException.ResultCodes getResult() {
+ return result;
+ }
+
+ /**
+ * Error codes to make it easy to decode these exceptions.
+ */
+ public enum ResultCodes {
+ OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST,
+ SECRET_MANAGER_HMAC_ERROR
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 5caebf5..9af8dd0 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -35,6 +35,7 @@ This is similar to Namenode for Ozone.
*/
import "hdds.proto";
+import "Security.proto";
enum Status {
OK = 1;
@@ -544,6 +545,24 @@ service OzoneManagerService {
rpc getServiceList(ServiceListRequest)
returns(ServiceListResponse);
+
+ /**
+ Returns delegation token signed by OzoneManager.
+ */
+ rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto)
+ returns(hadoop.common.GetDelegationTokenResponseProto);
+
+ /**
+ Renew delegation token signed by OzoneManager.
+ */
+ rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto)
+ returns(hadoop.common.RenewDelegationTokenResponseProto);
+ /**
+ Cancels a delegation token signed by OzoneManager.
+ */
+ rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto)
+ returns(hadoop.common.CancelDelegationTokenResponseProto);
+
/**
Creates an S3 bucket and creates an ozone mapping for that bucket.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59767be1/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
new file mode 100644
index 0000000..e4a8f2b
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.Signature;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link OzoneSecretManager}.
+ */
+public class TestOzoneSecretManager {
+
+ private OzoneSecretManager<OzoneTokenIdentifier> secretManager;
+ private SecurityConfig securityConfig;
+ private KeyPair keyPair;
+ private long expiryTime;
+ private Text serviceRpcAdd;
+ private OzoneConfiguration conf;
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneSecretManager.class.getSimpleName());
+ private final static Text TEST_USER = new Text("testUser");
+ private long tokenMaxLifetime = 1000 * 20;
+ private long tokenRemoverScanInterval = 1000 * 20;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+ securityConfig = new SecurityConfig(conf);
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+ serviceRpcAdd = new Text("localhost");
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ secretManager.stop();
+ FileUtils.deleteQuietly(new File(BASEDIR));
+ }
+
+ @Test
+ public void testCreateToken() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ OzoneTokenIdentifier identifier =
+ OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
+ // Check basic details.
+ Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
+ Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
+ Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
+
+ validateHash(token.getPassword(), token.getIdentifier());
+ }
+
+ @Test
+ public void testRenewTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(10 * 5);
+ long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
+ Assert.assertTrue(renewalTime > 0);
+ }
+
+ /**
+ * Tests failure for mismatch in renewer.
+ */
+ @Test
+ public void testRenewTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser tries to renew a token", () -> {
+ secretManager.renewToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Tests token renew failure due to max time.
+ */
+ @Test
+ public void testRenewTokenFailureMaxTime() throws Exception {
+ secretManager = createSecretManager(conf, 100,
+ 100, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(101);
+ LambdaTestUtils.intercept(IOException.class,
+ "testUser tried to renew an expired token", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ /**
+ * Tests token renew failure due to renewal time.
+ */
+ @Test
+ public void testRenewTokenFailureRenewalTime() throws Exception {
+ secretManager = createSecretManager(conf, 1000 * 10,
+ 10, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(15);
+ LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ @Test
+ public void testCreateIdentifier() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ OzoneTokenIdentifier identifier = secretManager.createIdentifier();
+ // Check basic details.
+ Assert.assertTrue(identifier.getOwner().equals(new Text("")));
+ Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
+ Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
+ }
+
+ @Test
+ public void testCancelTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ secretManager.cancelToken(token, TEST_USER.toString());
+ }
+
+ @Test
+ public void testCancelTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser is not authorized to cancel the token", () -> {
+ secretManager.cancelToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Validate hash using public key of KeyPair.
+ */
+ private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+ Signature rsaSignature =
+ Signature.getInstance(securityConfig.getSignatureAlgo(),
+ securityConfig.getProvider());
+ rsaSignature.initVerify(keyPair.getPublic());
+ rsaSignature.update(identifier);
+ Assert.assertTrue(rsaSignature.verify(hash));
+ }
+
+ /**
+ * Create instance of {@link OzoneSecretManager}.
+ */
+ private OzoneSecretManager<OzoneTokenIdentifier> createSecretManager(
+ OzoneConfiguration config, long tokenMaxLife, long expiry, long
+ tokenRemoverScanTime) throws IOException {
+ return new OzoneSecretManager<>(config, tokenMaxLife,
+ expiry, tokenRemoverScanTime, serviceRpcAdd);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-103. SCM CA: Add new security
protocol for SCM to expose security related functions. Contributed by Ajay
Kumar.
Posted by xy...@apache.org.
HDDS-103. SCM CA: Add new security protocol for SCM to expose security related functions. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0be8650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0be8650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0be8650
Branch: refs/heads/HDDS-4
Commit: d0be86504421594ed70080ad6bc4faba52c8e437
Parents: 1842ced
Author: Ajay Kumar <aj...@apache.com>
Authored: Sun Oct 28 22:44:41 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 1 +
.../hdds/protocol/SCMSecurityProtocol.java | 44 +++++++
...MSecurityProtocolClientSideTranslatorPB.java | 99 +++++++++++++++
.../hdds/protocolPB/SCMSecurityProtocolPB.java | 35 ++++++
...MSecurityProtocolServerSideTranslatorPB.java | 66 ++++++++++
.../hadoop/hdds/protocolPB/package-info.java | 22 ++++
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 16 +++
.../apache/hadoop/ozone/OzoneSecurityUtil.java | 43 +++++++
.../src/main/proto/SCMSecurityProtocol.proto | 66 ++++++++++
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 22 ++++
.../hdds/scm/server/SCMBlockProtocolServer.java | 2 +-
.../scm/server/SCMSecurityProtocolServer.java | 121 +++++++++++++++++++
.../scm/server/StorageContainerManager.java | 26 +++-
13 files changed, 558 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 061158c..25774f8 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -239,6 +239,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>StorageContainerLocationProtocol.proto</include>
<include>hdds.proto</include>
<include>ScmBlockLocationProtocol.proto</include>
+ <include>SCMSecurityProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
new file mode 100644
index 0000000..f0ae41c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public interface SCMSecurityProtocol {
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dataNodeDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ String getDataNodeCertificate(
+ DatanodeDetailsProto dataNodeDetails,
+ String certSignReq) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..1726282
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * This class is the client-side translator that forwards requests for
+ * {@link SCMSecurityProtocol} to the {@link SCMSecurityProtocolPB} proxy.
+ */
+public class SCMSecurityProtocolClientSideTranslatorPB implements
+ SCMSecurityProtocol, ProtocolTranslator, Closeable {
+
+ /**
+ * RpcController is not used and hence is set to null.
+ */
+ private static final RpcController NULL_RPC_CONTROLLER = null;
+ private final SCMSecurityProtocolPB rpcProxy;
+
+ public SCMSecurityProtocolClientSideTranslatorPB(
+ SCMSecurityProtocolPB rpcProxy) {
+ this.rpcProxy = rpcProxy;
+ }
+
+ /**
+ * Closes this stream and releases any system resources associated
+ * with it. If the stream is already closed then invoking this
+ * method has no effect.
+ *
+ * <p> As noted in {@link AutoCloseable#close()}, cases where the
+ * close may fail require careful attention. It is strongly advised
+ * to relinquish the underlying resources and to internally
+ * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+ * the {@code IOException}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ @Override
+ public void close() throws IOException {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dataNodeDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ @Override
+ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
+ String certSignReq) throws IOException {
+ SCMGetDataNodeCertRequestProto.Builder builder =
+ SCMGetDataNodeCertRequestProto
+ .newBuilder()
+ .setCSR(certSignReq)
+ .setDatanodeDetails(dataNodeDetails);
+ try {
+ return rpcProxy
+ .getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
+ .getX509Certificate();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ /**
+ * Return the proxy object underlying this protocol translator.
+ *
+ * @return the proxy object underlying this protocol translator.
+ */
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
new file mode 100644
index 0000000..cc60c0a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol for security related operations on SCM.
+ */
+
+@ProtocolInfo(protocolName =
+ "org.apache.hadoop.ozone.protocol.SCMSecurityProtocol",
+ protocolVersion = 1)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+public interface SCMSecurityProtocolPB extends
+ SCMSecurityProtocolService.BlockingInterface {
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..40b9652
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto.ResponseCode;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+
+/**
+ * This class is the server-side translator that forwards requests received on
+ * {@link SCMSecurityProtocolPB} to the {@link
+ * SCMSecurityProtocol} server implementation.
+ */
+public class SCMSecurityProtocolServerSideTranslatorPB implements
+ SCMSecurityProtocolPB {
+
+ private final SCMSecurityProtocol impl;
+
+ public SCMSecurityProtocolServerSideTranslatorPB(SCMSecurityProtocol impl) {
+ this.impl = impl;
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param controller
+ * @param request
+ * @return SCMGetDataNodeCertResponseProto.
+ */
+ @Override
+ public SCMGetDataNodeCertResponseProto getDataNodeCertificate(
+ RpcController controller, SCMGetDataNodeCertRequestProto request)
+ throws ServiceException {
+ try {
+ String certificate = impl
+ .getDataNodeCertificate(request.getDatanodeDetails(),
+ request.getCSR());
+ SCMGetDataNodeCertResponseProto.Builder builder =
+ SCMGetDataNodeCertResponseProto
+ .newBuilder()
+ .setResponseCode(ResponseCode.success)
+ .setX509Certificate(certificate);
+ return builder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
new file mode 100644
index 0000000..4496019
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.protocolPB;
+/**
+ * This package contains classes for wiring HDDS protobuf calls to rpc.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index abaa0f8..f17c7ec 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -159,6 +159,10 @@ public final class ScmConfigKeys {
"ozone.scm.block.client.port";
public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
+ public static final String OZONE_SCM_SECURITY_SERVICE_PORT_KEY =
+ "ozone.scm.security.service.port";
+ public static final int OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT = 9961;
+
// Container service client
public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
"ozone.scm.client.address";
@@ -175,6 +179,14 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
+ // SCM Security service address.
+ public static final String OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY =
+ "ozone.scm.security.service.address";
+ public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY =
+ "ozone.scm.security.service.bind.host";
+ public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT =
+ "0.0.0.0";
+
public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
"ozone.scm.datanode.address";
public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
@@ -217,6 +229,10 @@ public final class ScmConfigKeys {
"ozone.scm.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
+ public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
+ "ozone.scm.security.handler.count.key";
+ public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
+
public static final String OZONE_SCM_DEADNODE_INTERVAL =
"ozone.scm.dead.node.interval";
public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
new file mode 100644
index 0000000..d51d829
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Ozone security Util class.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneSecurityUtil {
+
+ private OzoneSecurityUtil() {
+ }
+
+ public static boolean isSecurityEnabled(Configuration conf) {
+ return conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+ OZONE_SECURITY_ENABLED_DEFAULT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
new file mode 100644
index 0000000..81a93c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.hdds.protocol.proto";
+
+option java_outer_classname = "SCMSecurityProtocolProtos";
+
+option java_generic_services = true;
+
+option java_generate_equals_and_hash = true;
+
+package hadoop.hdds;
+
+import "hdds.proto";
+
+/**
+ * Returns a certificate signed by SCM.
+ */
+message SCMGetDataNodeCertResponseProto {
+ enum ResponseCode {
+ success = 1;
+ authenticationFailed = 2;
+ invalidCSR = 3;
+ }
+ required ResponseCode responseCode = 1;
+ required string x509Certificate = 2; // Base64 encoded X509 certificate.
+}
+
+/**
+* This message is send by data node to prove its identity and get an SCM
+* signed certificate.
+*/
+message SCMGetDataNodeCertRequestProto {
+ required DatanodeDetailsProto datanodeDetails = 1;
+ required string CSR = 2;
+}
+
+
+service SCMSecurityProtocolService {
+ /**
+ * Get SCM signed certificate for DataNode.
+ */
+ rpc getDataNodeCertificate (SCMGetDataNodeCertRequestProto) returns (SCMGetDataNodeCertResponseProto);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 40b3d7a..e355d0a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -161,6 +161,28 @@ public final class HddsServerUtil {
}
/**
+ * Retrieve the socket address that should be used by scm security server to
+ * service clients.
+ *
+ * @param conf
+ * @return Target InetSocketAddress for the SCM security service.
+ */
+ public static InetSocketAddress getScmSecurityInetAddress(
+ Configuration conf) {
+ final Optional<String> host = getHostNameFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY);
+
+ final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
+ host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+ ":" + port
+ .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
+ }
+
+ /**
* Retrieve the socket address that should be used by DataNodes to connect
* to the SCM.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 3bb284e..2359c30 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -84,7 +84,7 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
ProtobufRpcEngine.class);
- // SCM Block Service RPC
+ // SCM Block Service RPC.
BlockingService blockProtoPbService =
ScmBlockLocationProtocolProtos.ScmBlockLocationProtocolService
.newReflectiveBlockingService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
new file mode 100644
index 0000000..e810c54
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.KerberosInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(SCMClientProtocolServer.class);
+ private final OzoneConfiguration config;
+ private final StorageContainerManager scm;
+ private final RPC.Server rpcServer;
+ private final InetSocketAddress rpcAddress;
+
+ SCMSecurityProtocolServer(OzoneConfiguration conf,
+ StorageContainerManager scm) throws IOException {
+ this.config = conf;
+ this.scm = scm;
+
+ final int handlerCount =
+ conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
+ rpcAddress = HddsServerUtil
+ .getScmSecurityInetAddress(conf);
+ // SCM security service RPC service.
+ BlockingService secureProtoPbService =
+ SCMSecurityProtocolProtos.SCMSecurityProtocolService
+ .newReflectiveBlockingService(
+ new SCMSecurityProtocolServerSideTranslatorPB(this));
+ this.rpcServer =
+ StorageContainerManager.startRpcServer(
+ conf,
+ rpcAddress,
+ SCMSecurityProtocolPB.class,
+ secureProtoPbService,
+ handlerCount);
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dnDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ @Override
+ public String getDataNodeCertificate(
+ DatanodeDetailsProto dnDetails,
+ String certSignReq) throws IOException {
+ LOGGER.info("Processing CSR for dn {}, UUID: {}", dnDetails.getHostName(),
+ dnDetails.getUuid());
+ // TODO: Call scm to sign the csr.
+ return null;
+ }
+
+ public RPC.Server getRpcServer() {
+ return rpcServer;
+ }
+
+ public InetSocketAddress getRpcAddress() {
+ return rpcAddress;
+ }
+
+ public void start() {
+ LOGGER.info(StorageContainerManager.buildRpcServerStartMessage("Starting"
+ + " RPC server for SCMSecurityProtocolServer.", getRpcAddress()));
+ getRpcServer().start();
+ }
+
+ public void stop() {
+ try {
+ LOGGER.info("Stopping the SCMSecurityProtocolServer.");
+ getRpcServer().stop();
+ } catch (Exception ex) {
+ LOGGER.error("SCMSecurityProtocolServer stop failed.", ex);
+ }
+ }
+
+ public void join() throws InterruptedException {
+ LOGGER.trace("Join RPC server for SCMSecurityProtocolServer.");
+ getRpcServer().join();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0be8650/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2d5b330..b25a402 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
@@ -106,8 +107,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
@@ -155,6 +154,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private final SCMDatanodeProtocolServer datanodeProtocolServer;
private final SCMBlockProtocolServer blockProtocolServer;
private final SCMClientProtocolServer clientProtocolServer;
+ private final SCMSecurityProtocolServer securityProtocolServer;
/*
* State Managers of SCM.
@@ -204,8 +204,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
StorageContainerManager.initMetrics();
initContainerReportCache(conf);
// Authenticate SCM if security is enabled
- if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
- OZONE_SECURITY_ENABLED_DEFAULT)) {
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
loginAsSCMUser(conf);
}
@@ -288,6 +287,11 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue);
blockProtocolServer = new SCMBlockProtocolServer(conf, this);
clientProtocolServer = new SCMClientProtocolServer(conf, this);
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
+ securityProtocolServer = new SCMSecurityProtocolServer(conf, this);
+ } else {
+ securityProtocolServer = null;
+ }
httpServer = new StorageContainerManagerHttpServer(conf);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
@@ -619,6 +623,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return clientProtocolServer;
}
+ public SCMSecurityProtocolServer getSecurityProtocolServer() {
+ return securityProtocolServer;
+ }
+
/**
* Initialize container reports cache that sent from datanodes.
*
@@ -721,6 +729,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
LOG.info(buildRpcServerStartMessage("ScmDatanodeProtocl RPC " +
"server", getDatanodeProtocolServer().getDatanodeRpcAddress()));
getDatanodeProtocolServer().start();
+ if(getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().start();
+ }
httpServer.start();
scmBlockManager.start();
@@ -785,6 +796,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
LOG.error("Storage Container Manager HTTP server stop failed.", ex);
}
+ if (getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().stop();
+ }
+
try {
LOG.info("Stopping Block Manager Service.");
scmBlockManager.stop();
@@ -821,6 +836,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
getBlockProtocolServer().join();
getClientProtocolServer().join();
getDatanodeProtocolServer().join();
+ if(getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().join();
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.info("Interrupted during StorageContainerManager join.");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDDS-760. Add asf license to
TestCertificateSignRequest. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-760. Add asf license to TestCertificateSignRequest. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd470bfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd470bfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd470bfd
Branch: refs/heads/HDDS-4
Commit: cd470bfd27c878783666ebb151941ae4c2eed812
Parents: d0be865
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 30 09:10:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../certificates/TestCertificateSignRequest.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd470bfd/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index a9285df..0b9ef31 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDDS-642. Add chill mode exit
condition for pipeline availability. Contributed by Yiqun Lin.
Posted by xy...@apache.org.
HDDS-642. Add chill mode exit condition for pipeline availability. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b71cc7f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b71cc7f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b71cc7f3
Branch: refs/heads/HDDS-4
Commit: b71cc7f33edbbf6a98d1efb330f1c748b5dd6e75
Parents: efc4d91
Author: Ajay Kumar <aj...@apache.org>
Authored: Wed Nov 28 17:45:46 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Wed Nov 28 17:47:57 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 9 ++
.../scm/chillmode/PipelineChillModeRule.java | 108 +++++++++++++++++++
.../hdds/scm/chillmode/SCMChillModeManager.java | 19 +++-
.../scm/server/StorageContainerManager.java | 5 +-
.../scm/chillmode/TestSCMChillModeManager.java | 81 ++++++++++++--
6 files changed, 213 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2d28a5b..f16503e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -87,6 +87,11 @@ public final class HddsConfigKeys {
"hdds.scm.chillmode.min.datanode";
public static final int HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT = 1;
+ public static final String HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK =
+ "hdds.scm.chillmode.pipeline-availability.check";
+ public static final boolean
+ HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = false;
+
// % of containers which should have at least one reported replica
// before SCM comes out of chill mode.
public static final String HDDS_SCM_CHILLMODE_THRESHOLD_PCT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 9f3d7e1..aa22b2b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1232,6 +1232,15 @@
</property>
<property>
+ <name>hdds.scm.chillmode.pipeline-availability.check</name>
+ <value>false</value>
+ <tag>HDDS,SCM,OPERATION</tag>
+ <description>
+ Boolean value to enable pipeline availability check during SCM chill mode.
+ </description>
+ </property>
+
+ <property>
<name>hdds.container.action.max.limit</name>
<value>20</value>
<tag>DATANODE</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
new file mode 100644
index 0000000..f9a6e59
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Class defining Chill mode exit criteria for Pipelines.
+ */
+public class PipelineChillModeRule
+ implements ChillModeExitRule<PipelineReportFromDatanode>,
+ EventHandler<PipelineReportFromDatanode> {
+ /** Pipeline availability.*/
+ private AtomicBoolean isPipelineAvailable = new AtomicBoolean(false);
+
+ private final PipelineManager pipelineManager;
+ private final SCMChillModeManager chillModeManager;
+
+ PipelineChillModeRule(PipelineManager pipelineManager,
+ SCMChillModeManager manager) {
+ this.pipelineManager = pipelineManager;
+ this.chillModeManager = manager;
+ }
+
+ @Override
+ public boolean validate() {
+ return isPipelineAvailable.get();
+ }
+
+ @Override
+ public void process(PipelineReportFromDatanode report) {
+ // No need to deal with
+ }
+
+ @Override
+ public void cleanup() {
+ // No need to deal with
+ }
+
+ @Override
+ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
+ EventPublisher publisher) {
+ // If we are already in pipeline available state,
+ // skipping following check.
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ return;
+ }
+
+ Pipeline pipeline;
+ Preconditions.checkNotNull(pipelineReportFromDatanode);
+ PipelineReportsProto pipelineReport = pipelineReportFromDatanode
+ .getReport();
+
+ for (PipelineReport report : pipelineReport.getPipelineReportList()) {
+ PipelineID pipelineID = PipelineID
+ .getFromProtobuf(report.getPipelineID());
+ try {
+ pipeline = pipelineManager.getPipeline(pipelineID);
+ } catch (PipelineNotFoundException e) {
+ continue;
+ }
+
+ if (pipeline.getPipelineState() == Pipeline.PipelineState.OPEN) {
+ // ensure there is an OPEN state pipeline and then allowed
+ // to exit chill mode
+ isPipelineAvailable.set(true);
+
+ if (chillModeManager.getInChillMode()) {
+ SCMChillModeManager.getLogger()
+ .info("SCM in chill mode. 1 Pipeline reported, 1 required.");
+ }
+ break;
+ }
+ }
+
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
index fa0eaa9..09b998d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
.NodeRegistrationContainerReport;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -53,14 +54,18 @@ public class SCMChillModeManager implements
LoggerFactory.getLogger(SCMChillModeManager.class);
private final boolean isChillModeEnabled;
private AtomicBoolean inChillMode = new AtomicBoolean(true);
+
private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
private Configuration config;
private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
+ private static final String PIPELINE_EXIT_RULE = "PipelineChillModeRule";
+
private final EventQueue eventPublisher;
public SCMChillModeManager(Configuration conf,
- List<ContainerInfo> allContainers, EventQueue eventQueue) {
+ List<ContainerInfo> allContainers, PipelineManager pipelineManager,
+ EventQueue eventQueue) {
this.config = conf;
this.eventPublisher = eventQueue;
this.isChillModeEnabled = conf.getBoolean(
@@ -70,6 +75,16 @@ public class SCMChillModeManager implements
exitRules.put(CONT_EXIT_RULE,
new ContainerChillModeRule(config, allContainers, this));
exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+
+ if (conf.getBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT)
+ && pipelineManager != null) {
+ PipelineChillModeRule rule = new PipelineChillModeRule(pipelineManager,
+ this);
+ exitRules.put(PIPELINE_EXIT_RULE, rule);
+ eventPublisher.addHandler(SCMEvents.PIPELINE_REPORT, rule);
+ }
emitChillModeStatus();
} else {
exitChillMode(eventQueue);
@@ -84,7 +99,7 @@ public class SCMChillModeManager implements
eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
}
- private void validateChillModeExitRules(EventPublisher eventQueue) {
+ public void validateChillModeExitRules(EventPublisher eventQueue) {
for (ChillModeExitRule exitRule : exitRules.values()) {
if (!exitRule.validate()) {
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fe00b54..2d27984 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -217,7 +217,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
NodeReportHandler nodeReportHandler =
new NodeReportHandler(scmNodeManager);
PipelineReportHandler pipelineReportHandler =
- new PipelineReportHandler(pipelineManager, conf);
+ new PipelineReportHandler(pipelineManager, conf);
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
@@ -302,8 +302,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
(BlockManagerImpl) scmBlockManager);
scmChillModeManager = new SCMChillModeManager(conf,
- containerManager.getContainers(),
- eventQueue);
+ containerManager.getContainers(), pipelineManager, eventQueue);
eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
index 0487fb7..ed17796 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -17,15 +17,29 @@
*/
package org.apache.hadoop.hdds.scm.chillmode;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
@@ -33,9 +47,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
/** Test class for SCMChillModeManager.
*/
public class TestSCMChillModeManager {
@@ -65,7 +76,7 @@ public class TestSCMChillModeManager {
@Test
public void testChillModeStateWithNullContainers() {
- new SCMChillModeManager(config, null, queue);
+ new SCMChillModeManager(config, null, null, queue);
}
private void testChillMode(int numContainers) throws Exception {
@@ -76,7 +87,8 @@ public class TestSCMChillModeManager {
for (ContainerInfo container : containers) {
container.setState(HddsProtos.LifeCycleState.OPEN);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -96,7 +108,8 @@ public class TestSCMChillModeManager {
for (ContainerInfo container : containers) {
container.setState(HddsProtos.LifeCycleState.CLOSED);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -118,7 +131,8 @@ public class TestSCMChillModeManager {
public void testDisableChillMode() {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ conf, containers, null, queue);
assertFalse(scmChillModeManager.getInChillMode());
}
@@ -149,7 +163,8 @@ public class TestSCMChillModeManager {
container.setState(HddsProtos.LifeCycleState.OPEN);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -173,7 +188,8 @@ public class TestSCMChillModeManager {
private void testChillModeDataNodes(int numOfDns) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ conf, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
// Assert SCM is in Chill mode.
@@ -212,4 +228,51 @@ public class TestSCMChillModeManager {
}, 100, 2000 * 9);
}
+ @Test
+ public void testChillModePipelineExitRule() throws Exception {
+ containers = new ArrayList<>();
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ String storageDir = GenericTestUtils.getTempPath(
+ TestSCMChillModeManager.class.getName() + UUID.randomUUID());
+ try{
+ MockNodeManager nodeManager = new MockNodeManager(true, 1);
+ config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ // enable pipeline check
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK, true);
+
+ PipelineManager pipelineManager = new SCMPipelineManager(config,
+ nodeManager, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, pipelineManager, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // simulation a pipeline report to trigger the rule check
+ Pipeline pipeline = pipelineManager.createPipeline(
+ HddsProtos.ReplicationType.STAND_ALONE,
+ HddsProtos.ReplicationFactor.ONE);
+ PipelineReportsProto.Builder reportBuilder = PipelineReportsProto
+ .newBuilder();
+ reportBuilder.addPipelineReport(PipelineReport.newBuilder()
+ .setPipelineID(pipeline.getId().getProtobuf()));
+
+ queue.fireEvent(SCMEvents.PIPELINE_REPORT, new PipelineReportFromDatanode(
+ pipeline.getNodes().get(0), reportBuilder.build()));
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 10);
+ pipelineManager.close();
+ } finally {
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+ false);
+ FileUtil.fullyDelete(new File(storageDir));
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8989. [YARN-8851] Move
DockerCommandPlugin volume related APIs' invocation from
DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun
Tang via wangda)
Posted by xy...@apache.org.
YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda)
Change-Id: Ia6d532c687168448416dfdf46f0ac34bff20e6ca
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7dab8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7dab8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7dab8e
Branch: refs/heads/HDDS-4
Commit: fe7dab8ef55f08cf18c2d62c782c1ab8930a5a15
Parents: 8976439
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:55:16 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 15:03:06 2018 -0800
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 44 ++++++++------------
.../runtime/TestDockerContainerRuntime.java | 15 ++++---
2 files changed, 24 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 15ff0ff..225bc19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -456,32 +456,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
-
- // Create volumes when needed.
- if (nmContext != null
- && nmContext.getResourcePluginManager().getNameToPlugins() != null) {
- for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
- .getNameToPlugins().values()) {
- DockerCommandPlugin dockerCommandPlugin =
- plugin.getDockerCommandPluginInstance();
- if (dockerCommandPlugin != null) {
- DockerVolumeCommand dockerVolumeCommand =
- dockerCommandPlugin.getCreateDockerVolumeCommand(
- ctx.getContainer());
- if (dockerVolumeCommand != null) {
- runDockerVolumeCommand(dockerVolumeCommand, container);
-
- // After volume created, run inspect to make sure volume properly
- // created.
- if (dockerVolumeCommand.getSubCommand().equals(
- DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
- checkDockerVolumeCreated(dockerVolumeCommand, container);
- }
- }
- }
- }
- }
}
private void checkDockerVolumeCreated(
@@ -1034,14 +1008,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
- // use plugins to update docker run command.
+ // use plugins to create volume and update docker run command.
if (nmContext != null
&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
.getNameToPlugins().values()) {
DockerCommandPlugin dockerCommandPlugin =
plugin.getDockerCommandPluginInstance();
+
if (dockerCommandPlugin != null) {
+ // Create volumes when needed.
+ DockerVolumeCommand dockerVolumeCommand =
+ dockerCommandPlugin.getCreateDockerVolumeCommand(
+ ctx.getContainer());
+ if (dockerVolumeCommand != null) {
+ runDockerVolumeCommand(dockerVolumeCommand, container);
+
+ // After volume created, run inspect to make sure volume properly
+ // created.
+ if (dockerVolumeCommand.getSubCommand().equals(
+ DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
+ checkDockerVolumeCreated(dockerVolumeCommand, container);
+ }
+ }
+ // Update cmd
dockerCommandPlugin.updateDockerRunCommand(runCommand, container);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 40ac618..0b5ce3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -1958,16 +1958,16 @@ public class TestDockerContainerRuntime {
ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
PrivilegedOperation.class);
- //single invocation expected
+ //Three invocations expected (volume creation, volume check, run container)
//due to type erasure + mocking, this verification requires a suppress
// warning annotation on the entire method
- verify(mockExecutor, times(2))
+ verify(mockExecutor, times(3))
.executePrivilegedOperation(anyList(), opCaptor.capture(), any(
File.class), anyMap(), anyBoolean(), anyBoolean());
//verification completed. we need to isolate specific invications.
// hence, reset mock here
- Mockito.reset(mockExecutor);
+ //Mockito.reset(mockExecutor);
List<PrivilegedOperation> allCaptures = opCaptor.getAllValues();
@@ -2070,10 +2070,8 @@ public class TestDockerContainerRuntime {
try {
runtime.prepareContainer(containerRuntimeContext);
-
- checkVolumeCreateCommand();
-
runtime.launchContainer(containerRuntimeContext);
+ checkVolumeCreateCommand();
} catch (ContainerExecutionException e) {
if (expectFail) {
// Expected
@@ -2166,10 +2164,11 @@ public class TestDockerContainerRuntime {
ContainerRuntimeContext containerRuntimeContext = builder.build();
runtime.prepareContainer(containerRuntimeContext);
- checkVolumeCreateCommand();
runtime.launchContainer(containerRuntimeContext);
- List<String> dockerCommands = readDockerCommands();
+ checkVolumeCreateCommand();
+
+ List<String> dockerCommands = readDockerCommands(3);
int expected = 14;
int counter = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-9067. YARN Resource Manager
is running OOM because of leak of Configuration Object. Contributed by Eric
Yang.
Posted by xy...@apache.org.
YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efc4d91c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efc4d91c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efc4d91c
Branch: refs/heads/HDDS-4
Commit: efc4d91cbeab8a13f6d61cb0e56443adb2d77559
Parents: fe7dab8
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 29 09:34:14 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 29 09:34:14 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/service/webapp/ApiServer.java | 209 +++++++++++--------
.../hadoop/yarn/service/ServiceClientTest.java | 2 +-
.../yarn/service/client/ServiceClient.java | 1 +
3 files changed, 126 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index db831ba..88aeefd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -118,10 +118,13 @@ public class ApiServer {
@Override
public Void run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.actionBuild(service);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionBuild(service);
+ } finally {
+ sc.close();
+ }
return null;
}
});
@@ -133,11 +136,14 @@ public class ApiServer {
@Override
public ApplicationId run() throws IOException, YarnException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId applicationId = sc.actionCreate(service);
- sc.close();
- return applicationId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId applicationId = sc.actionCreate(service);
+ return applicationId;
+ } finally {
+ sc.close();
+ }
}
});
serviceStatus.setDiagnostics("Application ID: " + applicationId);
@@ -245,29 +251,32 @@ public class ApiServer {
public Integer run() throws Exception {
int result = 0;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Exception stopException = null;
try {
- result = sc.actionStop(appName, destroy);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully stopped service {}", appName);
- }
- } catch (Exception e) {
- LOG.info("Got exception stopping service", e);
- stopException = e;
- }
- if (destroy) {
- result = sc.actionDestroy(appName);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully deleted service {}", appName);
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Exception stopException = null;
+ try {
+ result = sc.actionStop(appName, destroy);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ } catch (Exception e) {
+ LOG.info("Got exception stopping service", e);
+ stopException = e;
}
- } else {
- if (stopException != null) {
- throw stopException;
+ if (destroy) {
+ result = sc.actionDestroy(appName);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully deleted service {}", appName);
+ }
+ } else {
+ if (stopException != null) {
+ throw stopException;
+ }
}
+ } finally {
+ sc.close();
}
- sc.close();
return result;
}
});
@@ -378,13 +387,16 @@ public class ApiServer {
@Override
public Map<String, Long> run() throws YarnException, IOException {
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Map<String, Long> original = sc.flexByRestService(appName,
- Collections.singletonMap(componentName,
- component.getNumberOfContainers()));
- sc.close();
- return original;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Map<String, Long> original = sc.flexByRestService(appName,
+ Collections.singletonMap(componentName,
+ component.getNumberOfContainers()));
+ return original;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -632,12 +644,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionFlex(appName, componentCountStrings);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionFlex(appName, componentCountStrings);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -658,12 +673,15 @@ public class ApiServer {
@Override
public String run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- String newLifeTime = sc.updateLifetime(appName,
- updateAppData.getLifetime());
- sc.close();
- return newLifeTime;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ String newLifeTime = sc.updateLifetime(appName,
+ updateAppData.getLifetime());
+ return newLifeTime;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -681,11 +699,14 @@ public class ApiServer {
@Override public ApplicationId run()
throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId appId = sc.actionStartAndGetId(appName);
- sc.close();
- return appId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId appId = sc.actionStartAndGetId(appName);
+ return appId;
+ } finally {
+ sc.close();
+ }
}
});
LOG.info("Successfully started service " + appName);
@@ -702,14 +723,17 @@ public class ApiServer {
ServiceStatus status = new ServiceStatus();
ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
- sc.actionUpgradeExpress(service);
- } else {
- sc.initiateUpgrade(service);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
+ sc.actionUpgradeExpress(service);
+ } else {
+ sc.initiateUpgrade(service);
+ }
+ } finally {
+ sc.close();
}
- sc.close();
return null;
});
LOG.info("Service {} version {} upgrade initialized", service.getName(),
@@ -724,11 +748,14 @@ public class ApiServer {
final UserGroupInformation ugi) throws IOException, InterruptedException {
int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- int exitCode = sc.actionCancelUpgrade(serviceName);
- sc.close();
- return exitCode;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ int exitCode = sc.actionCancelUpgrade(serviceName);
+ return exitCode;
+ } finally {
+ sc.close();
+ }
});
if (result == EXIT_SUCCESS) {
ServiceStatus status = new ServiceStatus();
@@ -793,10 +820,13 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
int result1;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result1 = sc.actionUpgrade(service, containers);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result1 = sc.actionUpgrade(service, containers);
+ } finally {
+ sc.close();
+ }
return result1;
});
}
@@ -815,12 +845,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionDecommissionInstances(appName, instances);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionDecommissionInstances(appName, instances);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -840,11 +873,14 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Service app1 = sc.getStatus(serviceName);
- sc.close();
- return app1;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Service app1 = sc.getStatus(serviceName);
+ return app1;
+ } finally {
+ sc.close();
+ }
});
}
@@ -855,12 +891,15 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> {
ComponentContainers[] result;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc.getContainers(serviceName, componentNames, version,
- containerStates);
- sc.close();
- return result;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc.getContainers(serviceName, componentNames, version,
+ containerStates);
+ return result;
+ } finally {
+ sc.close();
+ }
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index d022614..89366b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient {
public void forceStop() {
expectedInstances.clear();
- super.stop();
+ stop();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 713d890..5cdb8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (registryClient != null) {
registryClient.stop();
}
+ fs.getFileSystem().close();
super.serviceStop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-546. Resolve bouncy castle
dependency for hadoop-hdds-common. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-546. Resolve bouncy castle dependency for hadoop-hdds-common. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1c6ff7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1c6ff7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1c6ff7e
Branch: refs/heads/HDDS-4
Commit: d1c6ff7ea3f4f02df0d766860187a391975475d1
Parents: b802cb5
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Sep 25 14:19:14 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c6ff7e/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 175500a..15ae307 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -59,6 +59,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -112,7 +116,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
- <version>1.49</version>
+ <version>1.54</version>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDDS-5. Enable OzoneManager
kerberos auth. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-5. Enable OzoneManager kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2119be45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2119be45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2119be45
Branch: refs/heads/HDDS-4
Commit: 2119be450b6516fe79278870effd13d542fb3d35
Parents: aa7ca15
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon May 14 09:36:57 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800
----------------------------------------------------------------------
.../common/src/main/resources/ozone-default.xml | 32 +++-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 9 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 6 +
.../om/protocolPB/OzoneManagerProtocolPB.java | 4 +
.../hadoop/ozone/MiniOzoneClusterImpl.java | 3 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 168 +++++++++++++++----
.../apache/hadoop/ozone/om/OzoneManager.java | 69 +++++++-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 5 +-
8 files changed, 246 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index a51443c..d3e352b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1400,7 +1400,23 @@
<name>ozone.scm.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+ <description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
+ </property>
+
+ <property>
+ <name>ozone.om.kerberos.keytab.file</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description> The keytab file used by KSM daemon to login as its
+ service principal. The principal name is configured with
+ hdds.ksm.kerberos.principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.kerberos.principal</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
</property>
<property>
@@ -1412,4 +1428,18 @@
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
+ <property>
+ <name>ozone.om.http.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <description>
+ KSM http server kerberos principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.http.kerberos.keytab.file</name>
+ <value>/etc/security/keytabs/HTTP.keytab</value>
+ <description>
+ KSM http server kerberos keytab.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 739c75e..0119eb5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -86,4 +86,13 @@ public final class OMConfigKeys {
public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
"ozone.om.save.metrics.interval";
public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
+
+ public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om."
+ + "kerberos.keytab.file";
+ public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ + ".kerberos.principal";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ "ozone.om.http.kerberos.keytab.file";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ = "ozone.om.http.kerberos.principal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index e4cce65..2a4e864 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.om.protocol;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -25,14 +26,19 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+
+
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol to talk to OM.
*/
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
public interface OzoneManagerProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index e0879d6..27e8f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.ozone.om.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used to communicate with OM.
@@ -28,6 +30,8 @@ import org.apache.hadoop.ozone.protocol.proto
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.OzoneManagerProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 7b0d73d..44cbecc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -468,7 +468,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
*
* @throws IOException
*/
- private OzoneManager createOM() throws IOException {
+ private OzoneManager createOM()
+ throws IOException, AuthenticationException {
configureOM();
OMStorage omStore = new OMStorage(conf);
initializeOmStorage(omStore);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 9c430ad..5c96067 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.Callable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,15 +36,22 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.security.KerberosAuthException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,13 +64,23 @@ public final class TestSecureOzoneCluster {
private Logger LOGGER = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
+ @Rule
+ public Timeout timeout = new Timeout(80000);
+
private MiniKdc miniKdc;
private OzoneConfiguration conf;
private File workDir;
private static Properties securityProperties;
private File scmKeytab;
private File spnegoKeytab;
+ private File omKeyTab;
private String curUser;
+ private StorageContainerManager scm;
+ private OzoneManager om;
+
+ private static String clusterId;
+ private static String scmId;
+ private static String omId;
@Before
public void init() {
@@ -71,6 +89,10 @@ public final class TestSecureOzoneCluster {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -78,12 +100,31 @@ public final class TestSecureOzoneCluster {
}
}
+ @After
+ public void stop() {
+ try {
+ stopMiniKdc();
+ if (scm != null) {
+ scm.stop();
+ }
+ if (om != null) {
+ om.stop();
+ }
+ } catch (Exception e) {
+ LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+ }
+ }
+
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -99,6 +140,10 @@ public final class TestSecureOzoneCluster {
miniKdc.start();
}
+ private void stopMiniKdc() throws Exception {
+ miniKdc.stop();
+ }
+
private void setSecureConfig(Configuration conf) throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
String host = KerberosUtil.getLocalHostName();
@@ -114,59 +159,56 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "om/" + host + "@" + realm);
+ conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_KSM/" + host + "@" + realm);
+
scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
+ omKeyTab = new File(workDir, "om.keytab");
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ omKeyTab.getAbsolutePath());
}
@Test
public void testSecureScmStartupSuccess() throws Exception {
+
+ initSCM();
+ scm = StorageContainerManager.createSCM(null, conf);
+ //Reads the SCM Info from SCM instance
+ ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+ Assert.assertEquals(clusterId, scmInfo.getClusterId());
+ Assert.assertEquals(scmId, scmInfo.getScmId());
+ }
+
+ private void initSCM()
+ throws IOException, AuthenticationException {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
- String clusterId = UUID.randomUUID().toString();
- String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
- StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
- //Reads the SCM Info from SCM instance
- ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
- Assert.assertEquals(clusterId, scmInfo.getClusterId());
- Assert.assertEquals(scmId, scmInfo.getScmId());
}
@Test
public void testSecureScmStartupFailure() throws Exception {
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- Path scmPath = Paths.get(path, "scm-meta");
-
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
- conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
- "scm@" + miniKdc.getRealm());
+ initSCM();
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
- SCMStorage scmStore = new SCMStorage(conf);
- String clusterId = UUID.randomUUID().toString();
- String scmId = UUID.randomUUID().toString();
- scmStore.setClusterId(clusterId);
- scmStore.setScmId(scmId);
- // writes the version file properties
- scmStore.initialize();
LambdaTestUtils.intercept(IOException.class,
"Running in secure mode, but config doesn't have a keytab",
() -> {
@@ -178,28 +220,82 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab");
+ testCommonKerberosFailures(
+ () -> StorageContainerManager.createSCM(null, conf));
+
+ }
+
+ private void testCommonKerberosFailures(Callable callable) throws Exception {
LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
- + "to login: for principal:",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ + "to login: for principal:", callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"OAuth2");
LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ " attribute value for hadoop.security.authentication of OAuth2",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"KERBEROS_SSL");
LambdaTestUtils.intercept(AuthenticationException.class,
- "KERBEROS_SSL authentication method not support.",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ "KERBEROS_SSL authentication method not",
+ callable);
+ }
+ /**
+ * Tests the secure KSM Initialization Failure.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationFailure() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ OMStorage ksmStore = new OMStorage(conf);
+ ksmStore.setClusterId("testClusterId");
+ ksmStore.setScmId("testScmId");
+ // writes the version file properties
+ ksmStore.initialize();
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "non-existent-user@EXAMPLE.com");
+ testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
+ }
+
+ /**
+ * Tests the secure KSM Initialization success.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationSuccess() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+ LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+ GenericTestUtils
+ .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
+ org.slf4j.event.Level.INFO);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path metaDirPath = Paths.get(path, "om-meta");
+
+ OMStorage omStore = new OMStorage(conf);
+ omStore.setClusterId("testClusterId");
+ omStore.setScmId("testScmId");
+ // writes the version file properties
+ omStore.initialize();
+ try {
+ om = OzoneManager.createOm(null, conf);
+ } catch (Exception ex) {
+ // Expects timeout failure from scmClient in KSM but KSM user login via
+ // kerberos should succeed
+ Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 71dde34..a05b2fd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -69,7 +70,10 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -115,6 +119,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys
import static org.apache.hadoop.ozone.om.OMConfigKeys
.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -123,7 +131,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
public final class OzoneManager extends ServiceRuntimeInfoImpl
implements OzoneManagerProtocol, OMMXBean, Auditor {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
private static final AuditLogger AUDIT =
@@ -161,14 +169,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
Preconditions.checkNotNull(conf);
configuration = conf;
omStorage = new OMStorage(conf);
- scmBlockClient = getScmBlockClient(configuration);
- scmContainerClient = getScmContainerClient(configuration);
if (omStorage.getState() != StorageState.INITIALIZED) {
throw new OMException("OM not initialized.",
ResultCodes.OM_NOT_INITIALIZED);
}
+ scmContainerClient = getScmContainerClient(configuration);
+
// verifies that the SCM info in the OM Version file is correct.
+ scmBlockClient = getScmBlockClient(configuration);
+
ScmInfo scmInfo = scmBlockClient.getScmInfo();
if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
.getScmId().equals(omStorage.getScmId()))) {
@@ -249,6 +259,35 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
+ * Login KSM service user if security and Kerberos are enabled.
+ *
+ * @param conf
+ * @throws IOException, AuthenticationException
+ */
+ private static void loginKSMUser(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
+
+ if (SecurityUtil.getAuthenticationMethod(conf).equals
+ (AuthenticationMethod.KERBEROS)) {
+ LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ + "Principal: {},keytab: {}", conf.get
+ (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
+
+ UserGroupInformation.setConfiguration(conf);
+
+ InetSocketAddress socAddr = getOmAddress(conf);
+ SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ } else {
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
+ (conf) + " authentication method not supported. KSM user login "
+ + "failed.");
+ }
+ LOG.info("KSM login successful.");
+ }
+
+ /**
* Create a scm block client, used by putKey() and getKey().
*
* @return {@link ScmBlockLocationProtocol}
@@ -365,15 +404,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @param argv Command line arguments
* @param conf OzoneConfiguration
* @return OM instance
- * @throws IOException in case OM instance creation fails.
+ * @throws IOException, AuthenticationException in case OM instance
+ * creation fails.
*/
@VisibleForTesting
public static OzoneManager createOm(
- String[] argv, OzoneConfiguration conf) throws IOException {
+ String[] argv, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createOm(argv, conf, false);
}
-
/**
* Constructs OM instance based on command line arguments.
*
@@ -381,10 +421,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @param conf OzoneConfiguration
* @param printBanner if true then log a verbose startup message.
* @return OM instance
- * @throws IOException in case OM instance creation fails.
+ * @throws IOException, AuthenticationException in case OM instance
+ * creation fails.
*/
private static OzoneManager createOm(String[] argv,
- OzoneConfiguration conf, boolean printBanner) throws IOException {
+ OzoneConfiguration conf, boolean printBanner)
+ throws IOException, AuthenticationException {
if (!isHddsEnabled(conf)) {
System.err.println("OM cannot be started in secure mode or when " +
OZONE_ENABLED + " is set to false");
@@ -396,6 +438,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
terminate(1);
return null;
}
+ // Authenticate KSM if security is enabled
+ if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+ loginKSMUser(conf);
+ }
switch (startOpt) {
case INIT:
if (printBanner) {
@@ -569,7 +615,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.start(configuration);
httpServer = new OzoneManagerHttpServer(configuration, this);
- httpServer.start();
+ try {
+ httpServer.start();
+ } catch (Exception ex) {
+ // Allow OM to start as Http Server failure is not fatal.
+ LOG.error("OM HttpServer failed to start.", ex);
+ }
registerMXBean();
setStartTime();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2119be45/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index bd6ab69..be12fa8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.server.BaseHttpServer;
@@ -65,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return OMConfigKeys.OZONE_OM_KEYTAB_FILE;
+ return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
}
@Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+ return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-759
Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-759
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18c9db98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18c9db98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18c9db98
Branch: refs/heads/HDDS-4
Commit: 18c9db980f8d3bcd01e3a9fa0f59861e26ffff89
Parents: 96bd574
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:21:44 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java | 2 +-
.../security/x509/certificates/TestCertificateSignRequest.java | 2 +-
.../hdds/security/x509/certificates/TestRootCertificate.java | 2 +-
.../hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 97627ca..2826e55 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -50,7 +50,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* A class that deals with all Security related configs in HDDS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index e8de1ea..9328c50 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -43,7 +43,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
public class TestCertificateSignRequest {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 16245a6..e581dc8 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -46,7 +46,7 @@ import java.time.Instant;
import java.util.Date;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* Test Class for Root Certificate generation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18c9db98/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
index f9541a2..08761f4 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.security.x509.keys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-704. Fix the Dependency
convergence issue on HDDS-4. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-704. Fix the Dependency convergence issue on HDDS-4. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/950a4733
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/950a4733
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/950a4733
Branch: refs/heads/HDDS-4
Commit: 950a4733ac86a0284ec46a1060ea3862f15181d3
Parents: b4c0280
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Oct 19 21:09:51 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/950a4733/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 30dcc58..061158c 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -112,13 +112,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>commons-pool2</artifactId>
<version>2.6.0</version>
</dependency>
-
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk15on</artifactId>
- <version>1.54</version>
- </dependency>
-
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed
Certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
HDDS-548. Create a Self-Signed Certificate. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87acc150
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87acc150
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87acc150
Branch: refs/heads/HDDS-4
Commit: 87acc1508a4bafc8cd3d77ffa684bf9871f52698
Parents: d1c6ff7
Author: Ajay Kumar <aj...@apache.com>
Authored: Fri Sep 28 06:52:56 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 16 ++
.../hdds/security/x509/HDDSKeyGenerator.java | 99 -------
.../hdds/security/x509/HDDSKeyPEMWriter.java | 254 ------------------
.../hdds/security/x509/SecurityConfig.java | 105 +++++---
.../certificates/SelfSignedCertificate.java | 212 +++++++++++++++
.../x509/certificates/package-info.java | 22 ++
.../x509/exceptions/CertificateException.java | 63 +++++
.../x509/exceptions/SCMSecurityException.java | 64 +++++
.../security/x509/exceptions/package-info.java | 23 ++
.../security/x509/keys/HDDSKeyGenerator.java | 106 ++++++++
.../security/x509/keys/HDDSKeyPEMWriter.java | 255 ++++++++++++++++++
.../hdds/security/x509/keys/package-info.java | 23 ++
.../security/x509/TestHDDSKeyGenerator.java | 81 ------
.../security/x509/TestHDDSKeyPEMWriter.java | 213 ---------------
.../x509/certificates/TestRootCertificate.java | 258 +++++++++++++++++++
.../x509/certificates/package-info.java | 22 ++
.../x509/keys/TestHDDSKeyGenerator.java | 87 +++++++
.../x509/keys/TestHDDSKeyPEMWriter.java | 216 ++++++++++++++++
.../hdds/security/x509/keys/package-info.java | 22 ++
.../hadoop/hdds/security/x509/package-info.java | 22 ++
20 files changed, 1484 insertions(+), 679 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2a1404a..ef1194b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -131,4 +131,20 @@ public final class HddsConfigKeys {
public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ ".name";
public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
+
+ /**
+ * Maximum duration of certificates issued by SCM including Self-Signed Roots.
+ * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
+ * Default value is 5 years and written as P1865D.
+ */
+ public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
+ // Limit Certificate duration to a max value of 5 years.
+ public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+
+ public static final String HDDS_X509_SIGNATURE_ALGO =
+ "hdds.x509.signature.algorithm";
+ public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
+
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
deleted file mode 100644
index cb411b2..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-
-/** A class to generate Key Pair for use with Certificates. */
-public class HDDSKeyGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(HDDSKeyGenerator.class);
- private final SecurityConfig securityConfig;
-
- /**
- * Constructor for HDDSKeyGenerator.
- *
- * @param configuration - config
- */
- public HDDSKeyGenerator(Configuration configuration) {
- this.securityConfig = new SecurityConfig(configuration);
- }
-
- /**
- * Returns the Security config used for this object.
- * @return SecurityConfig
- */
- public SecurityConfig getSecurityConfig() {
- return securityConfig;
- }
-
- /**
- * Use Config to generate key.
- *
- * @return KeyPair
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey() throws NoSuchProviderException,
- NoSuchAlgorithmException {
- return generateKey(securityConfig.getSize(),
- securityConfig.getAlgo(), securityConfig.getProvider());
- }
-
- /**
- * Specify the size -- all other parameters are used from config.
- *
- * @param size - int, valid key sizes.
- * @return KeyPair
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey(int size) throws
- NoSuchProviderException, NoSuchAlgorithmException {
- return generateKey(size,
- securityConfig.getAlgo(), securityConfig.getProvider());
- }
-
- /**
- * Custom Key Generation, all values are user provided.
- *
- * @param size - Key Size
- * @param algorithm - Algorithm to use
- * @param provider - Security provider.
- * @return KeyPair.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey(int size, String algorithm, String provider)
- throws NoSuchProviderException, NoSuchAlgorithmException {
- LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
- size, algorithm, provider);
- KeyPairGenerator generator = KeyPairGenerator
- .getInstance(algorithm, provider);
- generator.initialize(size);
- return generator.generateKeyPair();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
deleted file mode 100644
index 6ca7584..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.output.FileWriterWithEncoding;
-import org.apache.hadoop.conf.Configuration;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyPair;
-import java.util.Set;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * We store all Key material in good old PEM files.
- * This helps in avoiding dealing will persistent
- * Java KeyStore issues. Also when debugging,
- * general tools like OpenSSL can be used to read and
- * decode these files.
- */
-public class HDDSKeyPEMWriter {
- private static final Logger LOG =
- LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
- private final Path location;
- private final SecurityConfig securityConfig;
- private Set<PosixFilePermission> permissionSet =
- Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
- .collect(Collectors.toSet());
- private Supplier<Boolean> isPosixFileSystem;
- public final static String PRIVATE_KEY = "PRIVATE KEY";
- public final static String PUBLIC_KEY = "PUBLIC KEY";
- public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
- /*
- Creates an HDDS Key Writer.
-
- @param configuration - Configuration
- */
- public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration, "Config cannot be null");
- this.securityConfig = new SecurityConfig(configuration);
- isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
- this.location = securityConfig.getKeyLocation();
- }
-
- /**
- * Checks if File System supports posix style security permissions.
- *
- * @return True if it supports posix.
- */
- private static Boolean isPosix() {
- return FileSystems.getDefault().supportedFileAttributeViews()
- .contains("posix");
- }
-
- /**
- * Returns the Permission set.
- * @return Set
- */
- @VisibleForTesting
- public Set<PosixFilePermission> getPermissionSet() {
- return permissionSet;
- }
-
- /**
- * Returns the Security config used for this object.
- * @return SecurityConfig
- */
- public SecurityConfig getSecurityConfig() {
- return securityConfig;
- }
-
- /**
- * This function is used only for testing.
- *
- * @param isPosixFileSystem - Sets a boolean function for mimicking
- * files systems that are not posix.
- */
- @VisibleForTesting
- public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
- this.isPosixFileSystem = isPosixFileSystem;
- }
-
- /**
- * Writes a given key using the default config options.
- *
- * @param keyPair - Key Pair to write to file.
- * @throws IOException
- */
- public void writeKey(KeyPair keyPair) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), false);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException
- */
- public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), overwrite);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param basePath - The location to write to, override the config values.
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException
- */
- public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
- throws IOException {
- writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), overwrite);
- }
-
- /**
- * Helper function that actually writes data to the files.
- *
- * @param basePath - base path to write key
- * @param keyPair - Key pair to write to file.
- * @param privateKeyFileName - private key file name.
- * @param publicKeyFileName - public key file name.
- * @param force - forces overwriting the keys.
- * @throws IOException
- */
- private synchronized void writeKey(Path basePath, KeyPair keyPair,
- String privateKeyFileName, String publicKeyFileName, boolean force)
- throws IOException {
- checkPreconditions(basePath);
-
- File privateKeyFile =
- Paths.get(location.toString(), privateKeyFileName).toFile();
- File publicKeyFile =
- Paths.get(location.toString(), publicKeyFileName).toFile();
- checkKeyFile(privateKeyFile, force, publicKeyFile);
-
- try (PemWriter privateKeyWriter = new PemWriter(new
- FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
- privateKeyWriter.writeObject(
- new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
- }
-
- try (PemWriter publicKeyWriter = new PemWriter(new
- FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
- publicKeyWriter.writeObject(
- new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
- }
- Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
- Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
- }
-
- /**
- * Checks if private and public key file already exists. Throws IOException
- * if file exists and force flag is set to false, else will delete the
- * existing file.
- *
- * @param privateKeyFile - Private key file.
- * @param force - forces overwriting the keys.
- * @param publicKeyFile - public key file.
- * @throws IOException
- */
- private void checkKeyFile(File privateKeyFile, boolean force,
- File publicKeyFile) throws IOException {
- if (privateKeyFile.exists() && force) {
- if (!privateKeyFile.delete()) {
- throw new IOException("Unable to delete private key file.");
- }
- }
-
- if (publicKeyFile.exists() && force) {
- if (!publicKeyFile.delete()) {
- throw new IOException("Unable to delete public key file.");
- }
- }
-
- if (privateKeyFile.exists()) {
- throw new IOException("Private Key file already exists.");
- }
-
- if (publicKeyFile.exists()) {
- throw new IOException("Public Key file already exists.");
- }
- }
-
- /**
- * Checks if base path exists and sets file permissions.
- *
- * @param basePath - base path to write key
- * @throws IOException
- */
- private void checkPreconditions(Path basePath) throws IOException {
- Preconditions.checkNotNull(basePath, "Base path cannot be null");
- if (!isPosixFileSystem.get()) {
- LOG.error("Keys cannot be stored securely without POSIX file system "
- + "support for now.");
- throw new IOException("Unsupported File System for pem file.");
- }
-
- if (Files.exists(basePath)) {
- // Not the end of the world if we reset the permissions on an existing
- // directory.
- Files.setPosixFilePermissions(basePath, permissionSet);
- } else {
- boolean success = basePath.toFile().mkdirs();
- if (!success) {
- LOG.error("Unable to create the directory for the "
- + "location. Location: {}", basePath);
- throw new IOException("Unable to create the directory for the "
- + "location. Location:" + basePath);
- }
- Files.setPosixFilePermissions(basePath, permissionSet);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 896a379..9231c8a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -29,9 +29,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Provider;
import java.security.Security;
+import java.time.Duration;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
@@ -43,11 +44,17 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
/**
- * A class that deals with all Security related configs in HDDDS.
- * It is easier to have all Java code related to config in a single place.
+ * A class that deals with all Security related configs in HDDS.
+ *
+ * This class allows security configs to be read and used consistently across
+ * all of security related code base.
*/
public class SecurityConfig {
private static final Logger LOG =
@@ -55,15 +62,17 @@ public class SecurityConfig {
private static volatile Provider provider;
private final Configuration configuration;
private final int size;
- private final String algo;
+ private final String keyAlgo;
private final String providerString;
private final String metadatDir;
private final String keyDir;
- private final String privateKeyName;
- private final String publicKeyName;
+ private final String privateKeyFileName;
+ private final String publicKeyFileName;
+ private final Duration certDuration;
+ private final String x509SignatureAlgo;
/**
- * Constructs a HDDSKeyGenerator.
+ * Constructs a SecurityConfig.
*
* @param configuration - HDDS Configuration
*/
@@ -71,10 +80,10 @@ public class SecurityConfig {
Preconditions.checkNotNull(configuration, "Configuration cannot be null");
this.configuration = configuration;
this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
- this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+ this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
HDDS_DEFAULT_KEY_ALGORITHM);
this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
- HDDS_DEFAULT_SECURITY_PROVIDER);
+ HDDS_DEFAULT_SECURITY_PROVIDER);
// Please Note: To make it easy for our customers we will attempt to read
// HDDS metadata dir and if that is not set, we will use Ozone directory.
@@ -86,11 +95,17 @@ public class SecurityConfig {
+ " null. Please check configs.");
this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
HDDS_KEY_DIR_NAME_DEFAULT);
- this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+ this.privateKeyFileName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
- this.publicKeyName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+ this.publicKeyFileName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+ String durationString = this.configuration.get(HDDS_X509_MAX_DURATION,
+ HDDS_X509_MAX_DURATION_DEFAULT);
+ this.certDuration = Duration.parse(durationString);
+ this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
+ HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+
// First Startup -- if the provider is null, check for the provider.
if (SecurityConfig.provider == null) {
synchronized (SecurityConfig.class) {
@@ -105,39 +120,38 @@ public class SecurityConfig {
}
/**
- * Returns the Provider name.
- * @return String Provider name.
- */
- public String getProviderString() {
- return providerString;
- }
-
- /**
- * Returns the public key file name.
+ * Returns the public key file name, This is used for storing the public
+ * keys on disk.
+ *
* @return String, File name used for public keys.
*/
- public String getPublicKeyName() {
- return publicKeyName;
+ public String getPublicKeyFileName() {
+ return publicKeyFileName;
}
/**
- * Returns the private key file name.
+ * Returns the private key file name.This is used for storing the private
+ * keys on disk.
+ *
* @return String, File name used for private keys.
*/
- public String getPrivateKeyName() {
- return privateKeyName;
+ public String getPrivateKeyFileName() {
+ return privateKeyFileName;
}
/**
* Returns the File path to where keys are stored.
- * @return String Key location.
+ *
+ * @return String Key location.
*/
public Path getKeyLocation() {
return Paths.get(metadatDir, keyDir);
}
/**
- * Gets the Key Size.
+ * Gets the Key Size, The default key size is 2048, since the default
+ * algorithm used is RSA. User can change this by setting the "hdds.key
+ * .len" in configuration.
*
* @return key size.
*/
@@ -146,7 +160,8 @@ public class SecurityConfig {
}
/**
- * Gets provider.
+ * Returns the Provider name. SCM defaults to using Bouncy Castle and will
+ * return "BC".
*
* @return String Provider name.
*/
@@ -155,22 +170,48 @@ public class SecurityConfig {
}
/**
- * Returns the Key generation Algorithm used.
+ * Returns the Key generation Algorithm used. User can change this by
+ * setting the "hdds.key.algo" in configuration.
*
* @return String Algo.
*/
- public String getAlgo() {
- return algo;
+ public String getKeyAlgo() {
+ return keyAlgo;
+ }
+
+ /**
+ * Returns the X.509 Signature Algorithm used. This can be changed by setting
+ * "hdds.x509.signature.algorithm" to the new name. The default algorithm
+ * is SHA256withRSA.
+ *
+ * @return String
+ */
+ public String getSignatureAlgo() {
+ return x509SignatureAlgo;
}
/**
* Returns the Configuration used for initializing this SecurityConfig.
- * @return Configuration
+ *
+ * @return Configuration
*/
public Configuration getConfiguration() {
return configuration;
}
+ /**
+ * Returns the maximum length a certificate can be valid in SCM. The
+ * default value is 5 years. This can be changed by setting
+ * "hdds.x509.max.duration" in configuration. The formats accepted are
+ * based on the ISO-8601 duration format PnDTnHnMn.nS
+ *
+ * Default value is 5 years and written as P1865D.
+ *
+ * @return Duration.
+ */
+ public Duration getMaxCertificateDuration() {
+ return this.certDuration;
+ }
/**
* Adds a security provider dynamically if it is not loaded already.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
new file mode 100644
index 0000000..fef7ac3
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.util.Time;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.util.Date;
+
+/**
+ * A Self Signed Certificate with CA basic constraint can be used to boot-strap
+ * a certificate infra-structure, if no external certificate is provided.
+ */
+public final class SelfSignedCertificate {
+ private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private Date beginDate;
+ private Date endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+ private boolean isCA;
+
+ /**
+ * Private Ctor invoked only via Builder Interface.
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param beginDate - NotBefore
+ * @param endDate - Not After
+ * @param configuration - SCM Config
+ * @param keyPair - KeyPair
+ * @param ca - isCA?
+ */
+ private SelfSignedCertificate(String subject, String scmID, String clusterID,
+ Date beginDate, Date endDate, SecurityConfig configuration,
+ KeyPair keyPair, boolean ca) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.beginDate = beginDate;
+ this.endDate = endDate;
+ config = configuration;
+ this.key = keyPair;
+ this.isCA = ca;
+ }
+
+ @VisibleForTesting
+ public static String getNameFormat() {
+ return NAME_FORMAT;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ private X509CertificateHolder generateCertificate()
+ throws OperatorCreationException, CertIOException {
+ // For the Root Certificate we form the name from Subject, SCM ID and
+ // Cluster ID.
+ String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
+ X500Name name = new X500Name(dnName);
+ byte[] encoded = key.getPublic().getEncoded();
+ SubjectPublicKeyInfo publicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(encoded);
+
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(
+ config.getSignatureAlgo()).build(key.getPrivate());
+
+ // Please note: Since this is a root certificate we use "ONE" as the
+ // serial number. Also note that skip enforcing locale or UTC. We are
+ // trying to operate at the Days level, hence Time zone is also skipped for
+ // now.
+ BigInteger serial = BigInteger.ONE;
+ if (!isCA) {
+ serial = new BigInteger(Long.toString(Time.monotonicNow()));
+ }
+
+ X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
+ serial, beginDate, endDate, name, publicKeyInfo);
+
+ if (isCA) {
+ builder.addExtension(Extension.basicConstraints, true,
+ new BasicConstraints(true));
+ }
+ return builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Root Certificates.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private Date beginDate;
+ private Date endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+ private boolean isCA;
+
+ public Builder setConfiguration(Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ public Builder setBeginDate(Date date) {
+ this.beginDate = new Date(date.toInstant().toEpochMilli());
+ return this;
+ }
+
+ public Builder setEndDate(Date date) {
+ this.endDate = new Date(date.toInstant().toEpochMilli());
+ return this;
+ }
+
+ public Builder makeCA() {
+ isCA = true;
+ return this;
+ }
+
+ public X509CertificateHolder build() throws SCMSecurityException {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ Preconditions.checkArgument(beginDate.before(endDate), "Certificate " +
+ "begin date should be before end date");
+
+ Duration certDuration = Duration.between(beginDate.toInstant(),
+ endDate.toInstant());
+ Preconditions.checkArgument(
+ certDuration.compareTo(config.getMaxCertificateDuration()) < 0,
+ "Certificate life time cannot be greater than max configured value.");
+
+
+ SelfSignedCertificate rootCertificate =
+ new SelfSignedCertificate(this.subject,
+ this.scmID, this.clusterID, this.beginDate, this.endDate,
+ this.config, key, isCA);
+ try {
+ return rootCertificate.generateCertificate();
+ } catch (OperatorCreationException | CertIOException e) {
+ throw new CertificateException("Unable to create root certificate.",
+ e.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..e88737c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Utils for Certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
new file mode 100644
index 0000000..4d2dcfd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Certificate Exceptions from the SCM Security layer.
+ */
+public class CertificateException extends SCMSecurityException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public CertificateException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public CertificateException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public CertificateException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Error Message
+ * @param cause - Cause
+ * @param enableSuppression - Enable suppression.
+ * @param writableStackTrace - Writable stack trace.
+ */
+ public CertificateException(String message, Throwable cause,
+ boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
new file mode 100644
index 0000000..e1f6296
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends Exception {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public SCMSecurityException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public SCMSecurityException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public SCMSecurityException(Throwable cause) {
+ super(cause);
+ }
+
+
+ /**
+ * Ctor.
+ * @param message - Error Message
+ * @param cause - Cause
+ * @param enableSuppression - Enable suppression.
+ * @param writableStackTrace - Writable stack trace.
+ */
+ public SCMSecurityException(String message, Throwable cause,
+ boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
new file mode 100644
index 0000000..afcc474
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by X.509 security classes.
+ */
+package org.apache.hadoop.hdds.security.x509.exceptions;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
new file mode 100644
index 0000000..99873cb
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyGenerator.class);
+ private final SecurityConfig securityConfig;
+
+ /**
+ * Constructor for HDDSKeyGenerator.
+ *
+ * @param configuration - config
+ */
+ public HDDSKeyGenerator(Configuration configuration) {
+ this.securityConfig = new SecurityConfig(configuration);
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * Use Config to generate key.
+ *
+ * @return KeyPair
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ return generateKey(securityConfig.getSize(),
+ securityConfig.getKeyAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Specify the size -- all other parameters are used from config.
+ *
+ * @param size - int, valid key sizes.
+ * @return KeyPair
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey(int size) throws
+ NoSuchProviderException, NoSuchAlgorithmException {
+ return generateKey(size,
+ securityConfig.getKeyAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Custom Key Generation, all values are user provided.
+ *
+ * @param size - Key Size
+ * @param algorithm - Algorithm to use
+ * @param provider - Security provider.
+ * @return KeyPair.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey(int size, String algorithm, String provider)
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ size, algorithm, provider);
+ KeyPairGenerator generator = KeyPairGenerator
+ .getInstance(algorithm, provider);
+ generator.initialize(size);
+ return generator.generateKeyPair();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..95be1c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+ private final Path location;
+ private final SecurityConfig securityConfig;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ private Supplier<Boolean> isPosixFileSystem;
+ public final static String PRIVATE_KEY = "PRIVATE KEY";
+ public final static String PUBLIC_KEY = "PUBLIC KEY";
+ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+ /*
+ Creates an HDDS Key Writer.
+
+ @param configuration - Configuration
+ */
+ public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+ this.location = securityConfig.getKeyLocation();
+ }
+
+ /**
+ * Checks if File System supports posix style security permissions.
+ *
+ * @return True if it supports posix.
+ */
+ private static Boolean isPosix() {
+ return FileSystems.getDefault().supportedFileAttributeViews()
+ .contains("posix");
+ }
+
+ /**
+ * Returns the Permission set.
+ * @return Set
+ */
+ @VisibleForTesting
+ public Set<PosixFilePermission> getPermissionSet() {
+ return permissionSet;
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * This function is used only for testing.
+ *
+ * @param isPosixFileSystem - Sets a boolean function for mimicking
+ * files systems that are not posix.
+ */
+ @VisibleForTesting
+ public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+ this.isPosixFileSystem = isPosixFileSystem;
+ }
+
+ /**
+ * Writes a given key using the default config options.
+ *
+ * @param keyPair - Key Pair to write to file.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), false);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param basePath - The location to write to, override the config values.
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+ throws IOException {
+ writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Helper function that actually writes data to the files.
+ *
+ * @param basePath - base path to write key
+ * @param keyPair - Key pair to write to file.
+ * @param privateKeyFileName - private key file name.
+ * @param publicKeyFileName - public key file name.
+ * @param force - forces overwriting the keys.
+ * @throws IOException - On I/O failure.
+ */
+ private synchronized void writeKey(Path basePath, KeyPair keyPair,
+ String privateKeyFileName, String publicKeyFileName, boolean force)
+ throws IOException {
+ checkPreconditions(basePath);
+
+ File privateKeyFile =
+ Paths.get(location.toString(), privateKeyFileName).toFile();
+ File publicKeyFile =
+ Paths.get(location.toString(), publicKeyFileName).toFile();
+ checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+ try (PemWriter privateKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+ privateKeyWriter.writeObject(
+ new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+ }
+
+ try (PemWriter publicKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+ publicKeyWriter.writeObject(
+ new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+ }
+ Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+ Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Checks if private and public key file already exists. Throws IOException
+ * if file exists and force flag is set to false, else will delete the
+ * existing file.
+ *
+ * @param privateKeyFile - Private key file.
+ * @param force - forces overwriting the keys.
+ * @param publicKeyFile - public key file.
+ * @throws IOException - On I/O failure.
+ */
+ private void checkKeyFile(File privateKeyFile, boolean force,
+ File publicKeyFile) throws IOException {
+ if (privateKeyFile.exists() && force) {
+ if (!privateKeyFile.delete()) {
+ throw new IOException("Unable to delete private key file.");
+ }
+ }
+
+ if (publicKeyFile.exists() && force) {
+ if (!publicKeyFile.delete()) {
+ throw new IOException("Unable to delete public key file.");
+ }
+ }
+
+ if (privateKeyFile.exists()) {
+ throw new IOException("Private Key file already exists.");
+ }
+
+ if (publicKeyFile.exists()) {
+ throw new IOException("Public Key file already exists.");
+ }
+ }
+
+ /**
+ * Checks if base path exists and sets file permissions.
+ *
+ * @param basePath - base path to write key
+ * @throws IOException - On I/O failure.
+ */
+ private void checkPreconditions(Path basePath) throws IOException {
+ Preconditions.checkNotNull(basePath, "Base path cannot be null");
+ if (!isPosixFileSystem.get()) {
+ LOG.error("Keys cannot be stored securely without POSIX file system "
+ + "support for now.");
+ throw new IOException("Unsupported File System for pem file.");
+ }
+
+ if (Files.exists(basePath)) {
+ // Not the end of the world if we reset the permissions on an existing
+ // directory.
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ } else {
+ boolean success = basePath.toFile().mkdirs();
+ if (!success) {
+ LOG.error("Unable to create the directory for the "
+ + "location. Location: {}", basePath);
+ throw new IOException("Unable to create the directory for the "
+ + "location. Location:" + basePath);
+ }
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..37a04d6
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Utils for private and public keys.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
deleted file mode 100644
index 2ddf59c..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PublicKey;
-import java.security.interfaces.RSAPublicKey;
-import java.security.spec.PKCS8EncodedKeySpec;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for HDDS Key Generator.
- */
-public class TestHDDSKeyGenerator {
- private static SecurityConfig config;
-
- @Before
- public void init() {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
- config = new SecurityConfig(conf);
- }
- /**
- * In this test we verify that we are able to create a key pair, then get
- * bytes of that and use ASN1. parser to parse it back to a private key.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- @Test
- public void testGenerateKey()
- throws NoSuchProviderException, NoSuchAlgorithmException {
- HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
- KeyPair keyPair = keyGen.generateKey();
- Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
- PKCS8EncodedKeySpec keySpec =
- new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
- Assert.assertEquals("PKCS#8", keySpec.getFormat());
- }
-
- /**
- * In this test we assert that size that we specified is used for Key
- * generation.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- @Test
- public void testGenerateKeyWithSize() throws NoSuchProviderException,
- NoSuchAlgorithmException {
- HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
- KeyPair keyPair = keyGen.generateKey(4096);
- PublicKey publicKey = keyPair.getPublic();
- if(publicKey instanceof RSAPublicKey) {
- Assert.assertEquals(4096,
- ((RSAPublicKey)(publicKey)).getModulus().bitLength());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
deleted file mode 100644
index 68ff9e6..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test class for HDDS pem writer.
- */
-public class TestHDDSKeyPEMWriter {
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private OzoneConfiguration configuration;
- private HDDSKeyGenerator keyGenerator;
- private String prefix;
-
- @Before
- public void init() throws IOException {
- configuration = new OzoneConfiguration();
- prefix = temporaryFolder.newFolder().toString();
- configuration.set(HDDS_METADATA_DIR_NAME, prefix);
- keyGenerator = new HDDSKeyGenerator(configuration);
- }
-
- /**
- * Assert basic things like we are able to create a file, and the names are
- * in expected format etc.
- *
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- * @throws IOException
- */
- @Test
- public void testWriteKey()
- throws NoSuchProviderException, NoSuchAlgorithmException,
- IOException, InvalidKeySpecException {
- KeyPair keys = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.writeKey(keys);
-
- // Assert that locations have been created.
- Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
- Assert.assertTrue(keyLocation.toFile().exists());
-
- // Assert that locations are created in the locations that we specified
- // using the Config.
- Assert.assertTrue(keyLocation.toString().startsWith(prefix));
- Path privateKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPrivateKeyName());
- Assert.assertTrue(privateKeyPath.toFile().exists());
- Path publicKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPublicKeyName());
- Assert.assertTrue(publicKeyPath.toFile().exists());
-
- // Read the private key and test if the expected String in the PEM file
- // format exists.
- byte[] privateKey = Files.readAllBytes(privateKeyPath);
- String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
- Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
-
- // Read the public key and test if the expected String in the PEM file
- // format exists.
- byte[] publicKey = Files.readAllBytes(publicKeyPath);
- String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
- Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
-
- // Let us decode the PEM file and parse it back into binary.
- KeyFactory kf = KeyFactory.getInstance(
- pemWriter.getSecurityConfig().getAlgo());
-
- // Replace the PEM Human readable guards.
- privateKeydata =
- privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
- privateKeydata =
- privateKeydata.replace("-----END PRIVATE KEY-----", "");
-
- // Decode the bas64 to binary format and then use an ASN.1 parser to
- // parse the binary format.
-
- byte[] keyBytes = Base64.decodeBase64(privateKeydata);
- PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
- PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
- Assert.assertNotNull("Private Key should not be null",
- privateKeyDecoded);
-
- // Let us decode the public key and veriy that we can parse it back into
- // binary.
- publicKeydata =
- publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
- publicKeydata =
- publicKeydata.replace("-----END PUBLIC KEY-----", "");
-
- keyBytes = Base64.decodeBase64(publicKeydata);
- X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
- PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
- Assert.assertNotNull("Public Key should not be null",
- publicKeyDecoded);
-
- // Now let us assert the permissions on the Directories and files are as
- // expected.
- Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
- Set<PosixFilePermission> currentSet =
- Files.getPosixFilePermissions(privateKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(publicKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(keyLocation);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
- }
-
- /**
- * Assert key rewrite fails without force option.
- *
- * @throws IOException
- */
- @Test
- public void testReWriteKey()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- SecurityConfig secConfig = pemWriter.getSecurityConfig();
- pemWriter.writeKey(kp);
-
- // Assert that rewriting of keys throws exception with valid messages.
- LambdaTestUtils
- .intercept(IOException.class, "Private Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPrivateKeyName()).toFile());
- LambdaTestUtils
- .intercept(IOException.class, "Public Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPublicKeyName()).toFile());
-
- // Should succeed now as both public and private key are deleted.
- pemWriter.writeKey(kp);
- // Should succeed with overwrite flag as true.
- pemWriter.writeKey(kp, true);
-
- }
-
- /**
- * Assert key rewrite fails in non Posix file system.
- *
- * @throws IOException
- */
- @Test
- public void testWriteKeyInNonPosixFS()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.setIsPosixFileSystem(() -> false);
-
- // Assert key rewrite fails in non Posix file system.
- LambdaTestUtils
- .intercept(IOException.class, "Unsupported File System for pem file.",
- () -> pemWriter.writeKey(kp));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
new file mode 100644
index 0000000..3a00ca0
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SignatureException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * Test Class for Root Certificate generation.
+ */
+public class TestRootCertificate {
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testAllFieldsAreExpected()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException, CertificateException,
+ SignatureException, InvalidKeyException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ X509CertificateHolder certificateHolder = builder.build();
+
+ //Assert that we indeed have a self signed certificate.
+ Assert.assertEquals(certificateHolder.getIssuer(),
+ certificateHolder.getSubject());
+
+
+ // Make sure that NotBefore is before the current Date
+ Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+ Assert.assertFalse(
+ certificateHolder.getNotBefore()
+ .before(invalidDate));
+
+ //Make sure the end date is honored.
+ invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+ Assert.assertFalse(
+ certificateHolder.getNotAfter()
+ .after(invalidDate));
+
+ // Check the Subject Name and Issuer Name is in the expected format.
+ String dnName = String.format(SelfSignedCertificate.getNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
+ Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
+
+ // We did not ask for this Certificate to be a CA certificate, hence that
+ // extension should be null.
+ Assert.assertNull(
+ certificateHolder.getExtension(Extension.basicConstraints));
+
+ // Extract the Certificate and verify that certificate matches the public
+ // key.
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(keyPair.getPublic());
+ }
+
+ @Test
+ public void testCACert()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf)
+ .makeCA();
+
+ X509CertificateHolder certificateHolder = builder.build();
+ // This time we asked for a CA Certificate, make sure that extension is
+ // present and valid.
+ Extension basicExt =
+ certificateHolder.getExtension(Extension.basicConstraints);
+
+ Assert.assertNotNull(basicExt);
+ Assert.assertTrue(basicExt.isCritical());
+
+ // Since this code assigns ONE for the root certificate, we check if the
+ // serial number is the expected number.
+ Assert.assertEquals(certificateHolder.getSerialNumber(), BigInteger.ONE);
+ }
+
+ @Test
+ public void testInvalidParamFails()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setConfiguration(conf)
+ .setKey(keyPair)
+ .makeCA();
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with Blank Subject.
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+
+ // Swap the Begin and End Date and verify that we cannot create a
+ // certificate like that.
+ try {
+ builder.setBeginDate(notAfter);
+ builder.setEndDate(notBefore);
+ builder.build();
+ Assert.fail("Illegal dates should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setBeginDate(notBefore);
+ builder.setEndDate(notAfter);
+ }
+
+ try {
+ KeyPair newKey = keyGen.generateKey();
+ KeyPair wrongKey = new KeyPair(newKey.getPublic(), keyPair.getPrivate());
+ builder.setKey(wrongKey);
+ X509CertificateHolder certificateHolder = builder.build();
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(wrongKey.getPublic());
+ Assert.fail("Invalid Key, should have thrown.");
+ } catch (SCMSecurityException | CertificateException
+ | SignatureException | InvalidKeyException e) {
+ builder.setKey(keyPair);
+ }
+ // Assert that we can create a certificate with all sane params.
+ Assert.assertNotNull(builder.build());
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..c8a31fc
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test classes for Certificate utilities.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87acc150/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..f9541a2
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+ private SecurityConfig config;
+
+ @Before
+ public void init() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
+ config = new SecurityConfig(conf);
+ }
+ /**
+ * In this test we verify that we are able to create a key pair, then get
+ * bytes of that and use ASN1. parser to parse it back to a private key.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+ Assert.assertEquals(config.getKeyAlgo(),
+ keyPair.getPrivate().getAlgorithm());
+ PKCS8EncodedKeySpec keySpec =
+ new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+ Assert.assertEquals("PKCS#8", keySpec.getFormat());
+ }
+
+ /**
+ * In this test we assert that size that we specified is used for Key
+ * generation.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKeyWithSize() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey(4096);
+ PublicKey publicKey = keyPair.getPublic();
+ if(publicKey instanceof RSAPublicKey) {
+ Assert.assertEquals(4096,
+ ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDDS-873. Fix
TestSecureOzoneContainer NPE after HDDS-837. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-873. Fix TestSecureOzoneContainer NPE after HDDS-837. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b9b8e41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b9b8e41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b9b8e41
Branch: refs/heads/HDDS-4
Commit: 9b9b8e410f0d056ec3bae64da1f40013bbefde8b
Parents: 3bee23d
Author: Ajay Kumar <aj...@apache.org>
Authored: Mon Nov 26 15:49:01 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:48 2018 -0800
----------------------------------------------------------------------
.../container/ozoneimpl/TestSecureOzoneContainer.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b9b8e41/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
index 02d5e28..2224300 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -45,6 +47,7 @@ import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -133,7 +136,7 @@ public class TestSecureOzoneContainer {
OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
DatanodeDetails dn = TestUtils.randomDatanodeDetails();
- container = new OzoneContainer(dn, conf, null);
+ container = new OzoneContainer(dn, conf, getContext(dn));
//Setting scmId, as we start manually ozone container.
container.getDispatcher().setScmId(UUID.randomUUID().toString());
container.start();
@@ -206,4 +209,13 @@ public class TestSecureOzoneContainer {
Assert.assertNotNull(response);
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
}
+
+ private StateContext getContext(DatanodeDetails datanodeDetails) {
+ DatanodeStateMachine stateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ StateContext context = Mockito.mock(StateContext.class);
+ Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(stateMachine);
+ return context;
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-566. Move OzoneSecure
docker-compose after HDDS-447. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-566. Move OzoneSecure docker-compose after HDDS-447. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/585c3448
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/585c3448
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/585c3448
Branch: refs/heads/HDDS-4
Commit: 585c3448684cdae3c9f6c9c175c23038dfef80fc
Parents: eddbe99
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 2 10:07:35 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
hadoop-dist/src/main/compose/ozonesecure/.env | 18 ---
.../compose/ozonesecure/docker-compose.yaml | 57 -------
.../src/main/compose/ozonesecure/docker-config | 103 -------------
.../ozonesecure/docker-image/runner/Dockerfile | 39 -----
.../ozonesecure/docker-image/runner/build.sh | 26 ----
.../docker-image/runner/scripts/envtoconf.py | 115 --------------
.../docker-image/runner/scripts/krb5.conf | 38 -----
.../docker-image/runner/scripts/starter.sh | 100 -------------
.../runner/scripts/transformation.py | 150 -------------------
.../dist/src/main/compose/ozonesecure/.env | 18 +++
.../compose/ozonesecure/docker-compose.yaml | 57 +++++++
.../src/main/compose/ozonesecure/docker-config | 103 +++++++++++++
.../ozonesecure/docker-image/runner/Dockerfile | 39 +++++
.../ozonesecure/docker-image/runner/build.sh | 26 ++++
.../docker-image/runner/scripts/envtoconf.py | 115 ++++++++++++++
.../docker-image/runner/scripts/krb5.conf | 38 +++++
.../docker-image/runner/scripts/starter.sh | 100 +++++++++++++
.../runner/scripts/transformation.py | 150 +++++++++++++++++++
18 files changed, 646 insertions(+), 646 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
deleted file mode 100644
index a494004..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/.env
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
deleted file mode 100644
index 42ab05e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- kdc:
- image: ahadoop/kdc:v1
- hostname: kdc
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- datanode:
- image: ahadoop/runner:latest
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- hostname: datanode
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: ahadoop/runner:latest
- hostname: om
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: ahadoop/runner:latest
- hostname: scm
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
deleted file mode 100644
index 704dc7b..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-config
+++ /dev/null
@@ -1,103 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=5
-KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
deleted file mode 100644
index efda03e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License lsfor the specific language governing permissions and
-# limitations under the License.
-
-FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
-
-RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
-RUN chmod +x /usr/local/bin/dumb-init
-RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-ENV PATH $PATH:/opt/hadoop/bin
-
-RUN apt-get install -y jsvc
-ENV JSVC_HOME=/usr/bin
-ADD scripts/krb5.conf /etc/
-RUN apt-get install -y krb5-user
-
-RUN addgroup --gid 1000 hadoop
-RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
-RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-ADD scripts /opt/
-
-WORKDIR /opt/hadoop
-
-VOLUME /data
-ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
deleted file mode 100755
index abbdebe..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-set -e
-mkdir -p build
-if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
- wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
- cd "$DIR"/build || exit
- tar zvxf apache-rat.tar.gz
-fi
-java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
-docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
deleted file mode 100755
index ad2e176..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""convert environment variables to config"""
-
-import os
-import re
-
-import argparse
-
-import sys
-import transformation
-
-class Simple(object):
- """Simple conversion"""
- def __init__(self, args):
- parser = argparse.ArgumentParser()
- parser.add_argument("--destination", help="Destination directory", required=True)
- self.args = parser.parse_args(args=args)
- # copy the default files to file.raw in destination directory
-
- self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
- self.output_dir = self.args.destination
-
- self.configurables = {}
-
- def destination_file_path(self, name, extension):
- """destination file path"""
- return os.path.join(self.output_dir, "{}.{}".format(name, extension))
-
- def write_env_var(self, name, extension, key, value):
- """Write environment variables"""
- with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
- myfile.write("{}: {}\n".format(key, value))
-
- def process_envs(self):
- """Process environment variables"""
- for key in os.environ.keys():
- pattern = re.compile("[_\\.]")
- parts = pattern.split(key)
- extension = None
- name = parts[0].lower()
- if len(parts) > 1:
- extension = parts[1].lower()
- config_key = key[len(name) + len(extension) + 2:].strip()
- if extension and "!" in extension:
- splitted = extension.split("!")
- extension = splitted[0]
- fmt = splitted[1]
- config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
- else:
- fmt = extension
-
- if extension and extension in self.known_formats:
- if name not in self.configurables.keys():
- with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
- myfile.write("")
- self.configurables[name] = (extension, fmt)
- self.write_env_var(name, extension, config_key, os.environ[key])
- else:
- for configurable_name in self.configurables:
- if key.lower().startswith(configurable_name.lower()):
- self.write_env_var(configurable_name,
- self.configurables[configurable_name],
- key[len(configurable_name) + 1:],
- os.environ[key])
-
- def transform(self):
- """transform"""
- for configurable_name in self.configurables:
- name = configurable_name
- extension, fmt = self.configurables[name]
-
- destination_path = self.destination_file_path(name, extension)
-
- with open(destination_path + ".raw", "r") as myfile:
- content = myfile.read()
- transformer_func = getattr(transformation, "to_" + fmt)
- content = transformer_func(content)
- with open(destination_path, "w") as myfile:
- myfile.write(content)
-
- def main(self):
- """main"""
-
- # add the
- self.process_envs()
-
- # copy file.ext.raw to file.ext in the destination directory, and
- # transform to the right format (eg. key: value ===> XML)
- self.transform()
-
-
-def main():
- """main"""
- Simple(sys.argv[1:]).main()
-
-
-if __name__ == '__main__':
- Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
deleted file mode 100644
index 82ae73a..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_canonicalize_hostname = false
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- default_realm = EXAMPLE.COM
-
-[realms]
- EXAMPLE.COM = {
- kdc = SERVER
- admin_server = SERVER
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
deleted file mode 100755
index 04cd49d..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-set -e
-
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo "Setting up enviorment!!"
-
-if [ -n "$KERBEROS_ENABLED" ]; then
- echo "Setting up kerberos!!"
- KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
- ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
-
- echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
-
- if [ -n "$SLEEP_SECONDS" ]; then
- echo "Sleeping for $(SLEEP_SECONDS) seconds"
- sleep "$SLEEP_SECONDS"
- fi
-
-
- while true
- do
- STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
- if [ "$STATUS" -eq 200 ]; then
- echo "Got 200, KDC service ready!!"
- break
- else
- echo "Got $STATUS :( KDC service not ready yet..."
- fi
- sleep 5
- done
-
- HOST_NAME=$(hostname -f)
- export HOST_NAME
- for NAME in ${KERBEROS_KEYTABS}; do
- echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
- wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
- klist -kt "$CONF_DIR/$NAME.keytab"
- KERBEROS_ENABLED=true
- done
-
- sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
-fi
-
-#To avoid docker volume permission problems
-sudo chmod o+rwx /data
-
-"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
-
-if [ -n "$ENSURE_NAMENODE_DIR" ]; then
- CLUSTERID_OPTS=""
- if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
- CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
- fi
- if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
- /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
- fi
-fi
-
-if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
- if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
- /opt/hadoop/bin/hdfs namenode -bootstrapStandby
- fi
-fi
-
-if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
- if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
- /opt/hadoop/bin/ozone scm -init
- fi
-fi
-
-if [ -n "$ENSURE_OM_INITIALIZED" ]; then
- if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
- #To make sure SCM is running in dockerized environment we will sleep
- # Could be removed after HDFS-13203
- echo "Waiting 15 seconds for SCM startup"
- sleep 15
- /opt/hadoop/bin/ozone om -createObjectStore
- fi
-fi
-
-echo 'setup finished'
-"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
deleted file mode 100755
index 5e708ce..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""This module transform properties into different format"""
-def render_yaml(yaml_root, prefix=""):
- """render yaml"""
- result = ""
- if isinstance(yaml_root, dict):
- if prefix:
- result += "\n"
- for key in yaml_root:
- result += "{}{}: {}".format(prefix, key, render_yaml(
- yaml_root[key], prefix + " "))
- elif isinstance(yaml_root, list):
- result += "\n"
- for item in yaml_root:
- result += prefix + " - " + render_yaml(item, prefix + " ")
- else:
- result += "{}\n".format(yaml_root)
- return result
-
-
-def to_yaml(content):
- """transform to yaml"""
- props = process_properties(content)
-
- keys = props.keys()
- yaml_props = {}
- for key in keys:
- parts = key.split(".")
- node = yaml_props
- prev_part = None
- parent_node = {}
- for part in parts[:-1]:
- if part.isdigit():
- if isinstance(node, dict):
- parent_node[prev_part] = []
- node = parent_node[prev_part]
- while len(node) <= int(part):
- node.append({})
- parent_node = node
- node = node[int(node)]
- else:
- if part not in node:
- node[part] = {}
- parent_node = node
- node = node[part]
- prev_part = part
- if parts[-1].isdigit():
- if isinstance(node, dict):
- parent_node[prev_part] = []
- node = parent_node[prev_part]
- node.append(props[key])
- else:
- node[parts[-1]] = props[key]
-
- return render_yaml(yaml_props)
-
-
-def to_yml(content):
- """transform to yml"""
- return to_yaml(content)
-
-
-def to_properties(content):
- """transform to properties"""
- result = ""
- props = process_properties(content)
- for key, val in props.items():
- result += "{}: {}\n".format(key, val)
- return result
-
-
-def to_env(content):
- """transform to environment variables"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "{}={}\n".format(key, val)
- return result
-
-
-def to_sh(content):
- """transform to shell"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "export {}=\"{}\"\n".format(key, val)
- return result
-
-
-def to_cfg(content):
- """transform to config"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "{}={}\n".format(key, val)
- return result
-
-
-def to_conf(content):
- """transform to configuration"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "export {}={}\n".format(key, val)
- return result
-
-
-def to_xml(content):
- """transform to xml"""
- result = "<configuration>\n"
- props = process_properties(content)
- for key in props:
- result += "<property><name>{0}</name><value>{1}</value></property>\n". \
- format(key, props[key])
- result += "</configuration>"
- return result
-
-
-def process_properties(content, sep=': ', comment_char='#'):
- """
- Read the file passed as parameter as a properties file.
- """
- props = {}
- for line in content.split("\n"):
- sline = line.strip()
- if sline and not sline.startswith(comment_char):
- key_value = sline.split(sep)
- key = key_value[0].strip()
- value = sep.join(key_value[1:]).strip().strip('"')
- props[key] = value
-
- return props
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ image: ahadoop/kdc:v1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ image: ahadoop/runner:latest
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: ahadoop/runner:latest
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: ahadoop/runner:latest
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+ wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+ cd "$DIR"/build || exit
+ tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+ """Simple conversion"""
+ def __init__(self, args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--destination", help="Destination directory", required=True)
+ self.args = parser.parse_args(args=args)
+ # copy the default files to file.raw in destination directory
+
+ self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+ self.output_dir = self.args.destination
+
+ self.configurables = {}
+
+ def destination_file_path(self, name, extension):
+ """destination file path"""
+ return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+ def write_env_var(self, name, extension, key, value):
+ """Write environment variables"""
+ with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+ myfile.write("{}: {}\n".format(key, value))
+
+ def process_envs(self):
+ """Process environment variables"""
+ for key in os.environ.keys():
+ pattern = re.compile("[_\\.]")
+ parts = pattern.split(key)
+ extension = None
+ name = parts[0].lower()
+ if len(parts) > 1:
+ extension = parts[1].lower()
+ config_key = key[len(name) + len(extension) + 2:].strip()
+ if extension and "!" in extension:
+ splitted = extension.split("!")
+ extension = splitted[0]
+ fmt = splitted[1]
+ config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+ else:
+ fmt = extension
+
+ if extension and extension in self.known_formats:
+ if name not in self.configurables.keys():
+ with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+ myfile.write("")
+ self.configurables[name] = (extension, fmt)
+ self.write_env_var(name, extension, config_key, os.environ[key])
+ else:
+ for configurable_name in self.configurables:
+ if key.lower().startswith(configurable_name.lower()):
+ self.write_env_var(configurable_name,
+ self.configurables[configurable_name],
+ key[len(configurable_name) + 1:],
+ os.environ[key])
+
+ def transform(self):
+ """transform"""
+ for configurable_name in self.configurables:
+ name = configurable_name
+ extension, fmt = self.configurables[name]
+
+ destination_path = self.destination_file_path(name, extension)
+
+ with open(destination_path + ".raw", "r") as myfile:
+ content = myfile.read()
+ transformer_func = getattr(transformation, "to_" + fmt)
+ content = transformer_func(content)
+ with open(destination_path, "w") as myfile:
+ myfile.write(content)
+
+ def main(self):
+ """main"""
+
+ # add the
+ self.process_envs()
+
+ # copy file.ext.raw to file.ext in the destination directory, and
+ # transform to the right format (eg. key: value ===> XML)
+ self.transform()
+
+
+def main():
+ """main"""
+ Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+ Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = SERVER
+ admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+ echo "Setting up kerberos!!"
+ KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+ ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+ echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+ if [ -n "$SLEEP_SECONDS" ]; then
+ echo "Sleeping for $(SLEEP_SECONDS) seconds"
+ sleep "$SLEEP_SECONDS"
+ fi
+
+
+ while true
+ do
+ STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+ if [ "$STATUS" -eq 200 ]; then
+ echo "Got 200, KDC service ready!!"
+ break
+ else
+ echo "Got $STATUS :( KDC service not ready yet..."
+ fi
+ sleep 5
+ done
+
+ HOST_NAME=$(hostname -f)
+ export HOST_NAME
+ for NAME in ${KERBEROS_KEYTABS}; do
+ echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+ wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+ klist -kt "$CONF_DIR/$NAME.keytab"
+ KERBEROS_ENABLED=true
+ done
+
+ sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+ CLUSTERID_OPTS=""
+ if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+ CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+ fi
+ if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+ fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+ fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+ /opt/hadoop/bin/ozone scm -init
+ fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+ #To make sure SCM is running in dockerized environment we will sleep
+ # Could be removed after HDFS-13203
+ echo "Waiting 15 seconds for SCM startup"
+ sleep 15
+ /opt/hadoop/bin/ozone om -createObjectStore
+ fi
+fi
+
+echo 'setup finished'
+"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585c3448/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+ """render yaml"""
+ result = ""
+ if isinstance(yaml_root, dict):
+ if prefix:
+ result += "\n"
+ for key in yaml_root:
+ result += "{}{}: {}".format(prefix, key, render_yaml(
+ yaml_root[key], prefix + " "))
+ elif isinstance(yaml_root, list):
+ result += "\n"
+ for item in yaml_root:
+ result += prefix + " - " + render_yaml(item, prefix + " ")
+ else:
+ result += "{}\n".format(yaml_root)
+ return result
+
+
+def to_yaml(content):
+ """transform to yaml"""
+ props = process_properties(content)
+
+ keys = props.keys()
+ yaml_props = {}
+ for key in keys:
+ parts = key.split(".")
+ node = yaml_props
+ prev_part = None
+ parent_node = {}
+ for part in parts[:-1]:
+ if part.isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ while len(node) <= int(part):
+ node.append({})
+ parent_node = node
+ node = node[int(node)]
+ else:
+ if part not in node:
+ node[part] = {}
+ parent_node = node
+ node = node[part]
+ prev_part = part
+ if parts[-1].isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ node.append(props[key])
+ else:
+ node[parts[-1]] = props[key]
+
+ return render_yaml(yaml_props)
+
+
+def to_yml(content):
+ """transform to yml"""
+ return to_yaml(content)
+
+
+def to_properties(content):
+ """transform to properties"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props.items():
+ result += "{}: {}\n".format(key, val)
+ return result
+
+
+def to_env(content):
+ """transform to environment variables"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_sh(content):
+ """transform to shell"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}=\"{}\"\n".format(key, val)
+ return result
+
+
+def to_cfg(content):
+ """transform to config"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_conf(content):
+ """transform to configuration"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}={}\n".format(key, val)
+ return result
+
+
+def to_xml(content):
+ """transform to xml"""
+ result = "<configuration>\n"
+ props = process_properties(content)
+ for key in props:
+ result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+ format(key, props[key])
+ result += "</configuration>"
+ return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+ """
+ Read the file passed as parameter as a properties file.
+ """
+ props = {}
+ for line in content.split("\n"):
+ sline = line.strip()
+ if sline and not sline.startswith(comment_char):
+ key_value = sline.split(sep)
+ key = key_value[0].strip()
+ value = sep.join(key_value[1:]).strip().strip('"')
+ props[key] = value
+
+ return props
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDDS-100. SCM CA: generate
public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-100. SCM CA: generate public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed10fa6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed10fa6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed10fa6f
Branch: refs/heads/HDDS-4
Commit: ed10fa6fb160f210bcf424e151f2eeead34bf184
Parents: c8af727
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Jun 8 08:33:58 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:45 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 6 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 19 ++
.../hdds/security/x509/HDDSKeyGenerator.java | 99 ++++++++
.../hdds/security/x509/HDDSKeyPEMWriter.java | 254 +++++++++++++++++++
.../hdds/security/x509/SecurityConfig.java | 190 ++++++++++++++
.../hadoop/hdds/security/x509/package-info.java | 25 ++
.../common/src/main/resources/ozone-default.xml | 42 ++-
.../security/x509/TestHDDSKeyGenerator.java | 81 ++++++
.../security/x509/TestHDDSKeyPEMWriter.java | 213 ++++++++++++++++
.../ozone/TestOzoneConfigurationFields.java | 6 +
10 files changed, 933 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 0345012..175500a 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -81,7 +81,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>rocksdbjni</artifactId>
<version>5.14.2</version>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -110,6 +109,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>2.6.0</version>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>1.49</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index f16503e..2a1404a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -112,4 +112,23 @@ public final class HddsConfigKeys {
public static final String HDDS_PROMETHEUS_ENABLED =
"hdds.prometheus.endpoint.enabled";
+
+ public static final String HDDS_KEY_LEN = "hdds.key.len";
+ public static final int HDDS_DEFAULT_KEY_LEN = 2048;
+ public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
+ public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
+ public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
+ public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
+ public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
+ public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
+
+ // TODO : Talk to StorageIO classes and see if they can return a secure
+ // storage location for each node.
+ public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME =
+ "hdds.priv.key.file.name";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ + ".name";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
new file mode 100644
index 0000000..cb411b2
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyGenerator.class);
+ private final SecurityConfig securityConfig;
+
+ /**
+ * Constructor for HDDSKeyGenerator.
+ *
+ * @param configuration - config
+ */
+ public HDDSKeyGenerator(Configuration configuration) {
+ this.securityConfig = new SecurityConfig(configuration);
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * Use Config to generate key.
+ *
+ * @return KeyPair
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ return generateKey(securityConfig.getSize(),
+ securityConfig.getAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Specify the size -- all other parameters are used from config.
+ *
+ * @param size - int, valid key sizes.
+ * @return KeyPair
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey(int size) throws
+ NoSuchProviderException, NoSuchAlgorithmException {
+ return generateKey(size,
+ securityConfig.getAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Custom Key Generation, all values are user provided.
+ *
+ * @param size - Key Size
+ * @param algorithm - Algorithm to use
+ * @param provider - Security provider.
+ * @return KeyPair.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey(int size, String algorithm, String provider)
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ size, algorithm, provider);
+ KeyPairGenerator generator = KeyPairGenerator
+ .getInstance(algorithm, provider);
+ generator.initialize(size);
+ return generator.generateKeyPair();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..6ca7584
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+ private final Path location;
+ private final SecurityConfig securityConfig;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ private Supplier<Boolean> isPosixFileSystem;
+ public final static String PRIVATE_KEY = "PRIVATE KEY";
+ public final static String PUBLIC_KEY = "PUBLIC KEY";
+ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+ /*
+ Creates an HDDS Key Writer.
+
+ @param configuration - Configuration
+ */
+ public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+ this.location = securityConfig.getKeyLocation();
+ }
+
+ /**
+ * Checks if File System supports posix style security permissions.
+ *
+ * @return True if it supports posix.
+ */
+ private static Boolean isPosix() {
+ return FileSystems.getDefault().supportedFileAttributeViews()
+ .contains("posix");
+ }
+
+ /**
+ * Returns the Permission set.
+ * @return Set
+ */
+ @VisibleForTesting
+ public Set<PosixFilePermission> getPermissionSet() {
+ return permissionSet;
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * This function is used only for testing.
+ *
+ * @param isPosixFileSystem - Sets a boolean function for mimicking
+ * files systems that are not posix.
+ */
+ @VisibleForTesting
+ public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+ this.isPosixFileSystem = isPosixFileSystem;
+ }
+
+ /**
+ * Writes a given key using the default config options.
+ *
+ * @param keyPair - Key Pair to write to file.
+ * @throws IOException
+ */
+ public void writeKey(KeyPair keyPair) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), false);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException
+ */
+ public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), overwrite);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param basePath - The location to write to, override the config values.
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException
+ */
+ public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+ throws IOException {
+ writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), overwrite);
+ }
+
+ /**
+ * Helper function that actually writes data to the files.
+ *
+ * @param basePath - base path to write key
+ * @param keyPair - Key pair to write to file.
+ * @param privateKeyFileName - private key file name.
+ * @param publicKeyFileName - public key file name.
+ * @param force - forces overwriting the keys.
+ * @throws IOException
+ */
+ private synchronized void writeKey(Path basePath, KeyPair keyPair,
+ String privateKeyFileName, String publicKeyFileName, boolean force)
+ throws IOException {
+ checkPreconditions(basePath);
+
+ File privateKeyFile =
+ Paths.get(location.toString(), privateKeyFileName).toFile();
+ File publicKeyFile =
+ Paths.get(location.toString(), publicKeyFileName).toFile();
+ checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+ try (PemWriter privateKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+ privateKeyWriter.writeObject(
+ new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+ }
+
+ try (PemWriter publicKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+ publicKeyWriter.writeObject(
+ new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+ }
+ Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+ Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Checks if private and public key file already exists. Throws IOException
+ * if file exists and force flag is set to false, else will delete the
+ * existing file.
+ *
+ * @param privateKeyFile - Private key file.
+ * @param force - forces overwriting the keys.
+ * @param publicKeyFile - public key file.
+ * @throws IOException
+ */
+ private void checkKeyFile(File privateKeyFile, boolean force,
+ File publicKeyFile) throws IOException {
+ if (privateKeyFile.exists() && force) {
+ if (!privateKeyFile.delete()) {
+ throw new IOException("Unable to delete private key file.");
+ }
+ }
+
+ if (publicKeyFile.exists() && force) {
+ if (!publicKeyFile.delete()) {
+ throw new IOException("Unable to delete public key file.");
+ }
+ }
+
+ if (privateKeyFile.exists()) {
+ throw new IOException("Private Key file already exists.");
+ }
+
+ if (publicKeyFile.exists()) {
+ throw new IOException("Public Key file already exists.");
+ }
+ }
+
+ /**
+ * Checks if base path exists and sets file permissions.
+ *
+ * @param basePath - base path to write key
+ * @throws IOException
+ */
+ private void checkPreconditions(Path basePath) throws IOException {
+ Preconditions.checkNotNull(basePath, "Base path cannot be null");
+ if (!isPosixFileSystem.get()) {
+ LOG.error("Keys cannot be stored securely without POSIX file system "
+ + "support for now.");
+ throw new IOException("Unsupported File System for pem file.");
+ }
+
+ if (Files.exists(basePath)) {
+ // Not the end of the world if we reset the permissions on an existing
+ // directory.
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ } else {
+ boolean success = basePath.toFile().mkdirs();
+ if (!success) {
+ LOG.error("Unable to create the directory for the "
+ + "location. Location: {}", basePath);
+ throw new IOException("Unable to create the directory for the "
+ + "location. Location:" + basePath);
+ }
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
new file mode 100644
index 0000000..896a379
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.Provider;
+import java.security.Security;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * A class that deals with all Security related configs in HDDDS.
+ * It is easier to have all Java code related to config in a single place.
+ */
+public class SecurityConfig {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecurityConfig.class);
+ private static volatile Provider provider;
+ private final Configuration configuration;
+ private final int size;
+ private final String algo;
+ private final String providerString;
+ private final String metadatDir;
+ private final String keyDir;
+ private final String privateKeyName;
+ private final String publicKeyName;
+
+ /**
+ * Constructs a HDDSKeyGenerator.
+ *
+ * @param configuration - HDDS Configuration
+ */
+ public SecurityConfig(Configuration configuration) {
+ Preconditions.checkNotNull(configuration, "Configuration cannot be null");
+ this.configuration = configuration;
+ this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
+ this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+ HDDS_DEFAULT_KEY_ALGORITHM);
+ this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
+ HDDS_DEFAULT_SECURITY_PROVIDER);
+
+ // Please Note: To make it easy for our customers we will attempt to read
+ // HDDS metadata dir and if that is not set, we will use Ozone directory.
+ // TODO: We might want to fix this later.
+ this.metadatDir = this.configuration.get(HDDS_METADATA_DIR_NAME,
+ configuration.get(OZONE_METADATA_DIRS));
+
+ Preconditions.checkNotNull(this.metadatDir, "Metadata directory can't be"
+ + " null. Please check configs.");
+ this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
+ HDDS_KEY_DIR_NAME_DEFAULT);
+ this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+ HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
+ this.publicKeyName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+ HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+
+ // First Startup -- if the provider is null, check for the provider.
+ if (SecurityConfig.provider == null) {
+ synchronized (SecurityConfig.class) {
+ provider = Security.getProvider(this.providerString);
+ if (SecurityConfig.provider == null) {
+ // Provider not found, let us try to Dynamically initialize the
+ // provider.
+ provider = initSecurityProvider(this.providerString);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the Provider name.
+ * @return String Provider name.
+ */
+ public String getProviderString() {
+ return providerString;
+ }
+
+ /**
+ * Returns the public key file name.
+ * @return String, File name used for public keys.
+ */
+ public String getPublicKeyName() {
+ return publicKeyName;
+ }
+
+ /**
+ * Returns the private key file name.
+ * @return String, File name used for private keys.
+ */
+ public String getPrivateKeyName() {
+ return privateKeyName;
+ }
+
+ /**
+ * Returns the File path to where keys are stored.
+ * @return String Key location.
+ */
+ public Path getKeyLocation() {
+ return Paths.get(metadatDir, keyDir);
+ }
+
+ /**
+ * Gets the Key Size.
+ *
+ * @return key size.
+ */
+ public int getSize() {
+ return size;
+ }
+
+ /**
+ * Gets provider.
+ *
+ * @return String Provider name.
+ */
+ public String getProvider() {
+ return providerString;
+ }
+
+ /**
+ * Returns the Key generation Algorithm used.
+ *
+ * @return String Algo.
+ */
+ public String getAlgo() {
+ return algo;
+ }
+
+ /**
+ * Returns the Configuration used for initializing this SecurityConfig.
+ * @return Configuration
+ */
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+
+ /**
+ * Adds a security provider dynamically if it is not loaded already.
+ *
+ * @param providerName - name of the provider.
+ */
+ private Provider initSecurityProvider(String providerName) {
+ switch (providerName) {
+ case "BC":
+ Security.addProvider(new BouncyCastleProvider());
+ return Security.getProvider(providerName);
+ default:
+ LOG.error("Security Provider:{} is unknown", provider);
+ throw new SecurityException("Unknown security provider:" + provider);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..89d5d51
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+/**
+ * This package contains common routines used in creating an x509 based
+ * identity framework for HDDS.
+ */
+package org.apache.hadoop.hdds.security.x509;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 13b47c9..0c99368 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1451,4 +1451,44 @@
OzoneManager http server kerberos keytab.
</description>
</property>
-</configuration>
+ <property>
+ <name>hdds.key.len</name>
+ <value>2048</value>
+ <tag>SCM, HDDS, X509, SECURITY</tag>
+ <description>
+ SCM CA key length. This is an algorithm-specific metric, such as modulus length, specified in number of bits.
+ </description>
+ </property>
+ <property>
+ <name>hdds.key.dir.name</name>
+ <value>keys</value>
+ <tag>SCM, HDDS, X509, SECURITY</tag>
+ <description>
+ Directory to store public/private key for SCM CA. This is relative to ozone/hdds meteadata dir.
+ </description>
+ </property>
+ <property>
+ <name>hdds.metadata.dir</name>
+ <value/>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Absolute path to HDDS metadata dir.
+ </description>
+ </property>
+ <property>
+ <name>hdds.priv.key.file.name</name>
+ <value>private.pem</value>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Name of file which stores private key generated for SCM CA.
+ </description>
+ </property>
+ <property>
+ <name>hdds.public.key.file.name</name>
+ <value>public.pem</value>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Name of file which stores public key generated for SCM CA.
+ </description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..2ddf59c
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+ private static SecurityConfig config;
+
+ @Before
+ public void init() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
+ config = new SecurityConfig(conf);
+ }
+ /**
+ * In this test we verify that we are able to create a key pair, then get
+ * bytes of that and use ASN1. parser to parse it back to a private key.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ @Test
+ public void testGenerateKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+ Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
+ PKCS8EncodedKeySpec keySpec =
+ new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+ Assert.assertEquals("PKCS#8", keySpec.getFormat());
+ }
+
+ /**
+ * In this test we assert that size that we specified is used for Key
+ * generation.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ @Test
+ public void testGenerateKeyWithSize() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey(4096);
+ PublicKey publicKey = keyPair.getPublic();
+ if(publicKey instanceof RSAPublicKey) {
+ Assert.assertEquals(4096,
+ ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..68ff9e6
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OzoneConfiguration configuration;
+ private HDDSKeyGenerator keyGenerator;
+ private String prefix;
+
+ @Before
+ public void init() throws IOException {
+ configuration = new OzoneConfiguration();
+ prefix = temporaryFolder.newFolder().toString();
+ configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+ keyGenerator = new HDDSKeyGenerator(configuration);
+ }
+
+ /**
+ * Assert basic things like we are able to create a file, and the names are
+ * in expected format etc.
+ *
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ * @throws IOException
+ */
+ @Test
+ public void testWriteKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, InvalidKeySpecException {
+ KeyPair keys = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.writeKey(keys);
+
+ // Assert that locations have been created.
+ Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+ Assert.assertTrue(keyLocation.toFile().exists());
+
+ // Assert that locations are created in the locations that we specified
+ // using the Config.
+ Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+ Path privateKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPrivateKeyName());
+ Assert.assertTrue(privateKeyPath.toFile().exists());
+ Path publicKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPublicKeyName());
+ Assert.assertTrue(publicKeyPath.toFile().exists());
+
+ // Read the private key and test if the expected String in the PEM file
+ // format exists.
+ byte[] privateKey = Files.readAllBytes(privateKeyPath);
+ String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+ // Read the public key and test if the expected String in the PEM file
+ // format exists.
+ byte[] publicKey = Files.readAllBytes(publicKeyPath);
+ String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+ // Let us decode the PEM file and parse it back into binary.
+ KeyFactory kf = KeyFactory.getInstance(
+ pemWriter.getSecurityConfig().getAlgo());
+
+ // Replace the PEM Human readable guards.
+ privateKeydata =
+ privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+ privateKeydata =
+ privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+ // Decode the bas64 to binary format and then use an ASN.1 parser to
+ // parse the binary format.
+
+ byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+ PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+ PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+ Assert.assertNotNull("Private Key should not be null",
+ privateKeyDecoded);
+
+ // Let us decode the public key and veriy that we can parse it back into
+ // binary.
+ publicKeydata =
+ publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+ publicKeydata =
+ publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+ keyBytes = Base64.decodeBase64(publicKeydata);
+ X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+ PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+ Assert.assertNotNull("Public Key should not be null",
+ publicKeyDecoded);
+
+ // Now let us assert the permissions on the Directories and files are as
+ // expected.
+ Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+ Set<PosixFilePermission> currentSet =
+ Files.getPosixFilePermissions(privateKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(publicKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(keyLocation);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+ }
+
+ /**
+ * Assert key rewrite fails without force option.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testReWriteKey()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ SecurityConfig secConfig = pemWriter.getSecurityConfig();
+ pemWriter.writeKey(kp);
+
+ // Assert that rewriting of keys throws exception with valid messages.
+ LambdaTestUtils
+ .intercept(IOException.class, "Private Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPrivateKeyName()).toFile());
+ LambdaTestUtils
+ .intercept(IOException.class, "Public Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPublicKeyName()).toFile());
+
+ // Should succeed now as both public and private key are deleted.
+ pemWriter.writeKey(kp);
+ // Should succeed with overwrite flag as true.
+ pemWriter.writeKey(kp, true);
+
+ }
+
+ /**
+ * Assert key rewrite fails in non Posix file system.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testWriteKeyInNonPosixFS()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.setIsPosixFileSystem(() -> false);
+
+ // Assert key rewrite fails in non Posix file system.
+ LambdaTestUtils
+ .intercept(IOException.class, "Unsupported File System for pem file.",
+ () -> pemWriter.writeKey(kp));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed10fa6f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 986618c..7b59903 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -38,5 +38,11 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");
+ addPropertiesNotInXml();
+ }
+
+ private void addPropertiesNotInXml() {
+ configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_KEY_ALGORITHM);
+ configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_SECURITY_PROVIDER);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth.
Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa7ca153
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa7ca153
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa7ca153
Branch: refs/heads/HDDS-4
Commit: aa7ca153d8a9c9c9381f29f9ed739a56416db015
Parents: ae5fbdd
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800
----------------------------------------------------------------------
.../authentication/util/KerberosUtil.java | 2 +-
.../conf/TestConfigurationFieldsBase.java | 2 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 13 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +
.../StorageContainerLocationProtocol.java | 4 +
.../protocolPB/ScmBlockLocationProtocolPB.java | 6 +
.../StorageContainerLocationProtocolPB.java | 4 +
.../apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../common/src/main/resources/ozone-default.xml | 26 ++-
.../StorageContainerDatanodeProtocol.java | 4 +
.../StorageContainerDatanodeProtocolPB.java | 6 +
.../scm/server/StorageContainerManager.java | 51 ++++-
.../StorageContainerManagerHttpServer.java | 5 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 3 +
hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +-
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 13 +-
hadoop-ozone/integration-test/pom.xml | 6 +
.../TestContainerStateManagerIntegration.java | 5 +-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 4 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 21 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 205 +++++++++++++++++++
.../ozone/TestStorageContainerManager.java | 8 +-
24 files changed, 375 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
index c011045..4459928 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
@@ -167,7 +167,7 @@ public class KerberosUtil {
}
/* Return fqdn of the current host */
- static String getLocalHostName() throws UnknownHostException {
+ public static String getLocalHostName() throws UnknownHostException {
return InetAddress.getLocalHost().getCanonicalHostName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 152159b..bce1cd5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,6 +436,8 @@ public abstract class TestConfigurationFieldsBase {
// Create XML key/value map
LOG_XML.debug("Reading XML property files\n");
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+ // Remove hadoop property set in ozone-default.xml
+ xmlKeyValueMap.remove("hadoop.custom.tags");
LOG_XML.debug("\n=====\n");
// Create default configuration variable key/value map
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 18637af..fd6a0e3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -278,18 +278,7 @@ public final class HddsUtils {
}
public static boolean isHddsEnabled(Configuration conf) {
- String securityEnabled =
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- boolean securityAuthorizationEnabled = conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false);
-
- if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
- LOG.error("Ozone is not supported in a security enabled cluster. ");
- return false;
- } else {
- return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
- }
+ return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 062b101..e18fe91 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -192,8 +192,9 @@ public final class ScmConfigKeys {
"ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address";
- public static final String OZONE_SCM_KEYTAB_FILE =
- "ozone.scm.keytab.file";
+ public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
+ "ozone.scm.kerberos.keytab.file";
+ public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
@@ -312,6 +313,10 @@ public final class ScmConfigKeys {
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
+ public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+ "ozone.scm.web.authentication.kerberos.principal";
+ public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+ "ozone.scm.web.authentication.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index c8d4a80..e17f1c2 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdds.scm.protocol;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -31,6 +33,7 @@ import java.util.List;
* ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
* to read/write a block.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index c8a9dcf..5bc2521 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.protocol;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -27,11 +29,13 @@ import org.apache.hadoop.hdds.protocol.proto
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerLocationProtocol {
/**
* Asks SCM where a container should be allocated. SCM responds with the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 837c95b..89bb066 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.hdds.scm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
.ScmBlockLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from an HDFS node to StorageContainerManager. This extends the
@@ -30,6 +34,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
"org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
protocolVersion = 1)
@InterfaceAudience.Private
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocolPB
extends ScmBlockLocationProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index f234ad3..3bd83f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -21,7 +21,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos
.StorageContainerLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from an HDFS node to StorageContainerManager. This extends the
@@ -30,6 +32,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerLocationProtocolPB
extends StorageContainerLocationProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index df233f7..b47113b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -328,6 +328,10 @@ public final class OzoneConfigKeys {
public static final double
HDDS_DATANODE_STORAGE_UTILIZATION_CRITICAL_THRESHOLD_DEFAULT = 0.75;
+ public static final String OZONE_SECURITY_ENABLED_KEY =
+ "ozone.security.enabled";
+ public static final boolean OZONE_SECURITY_ENABLED_DEFAULT = false;
+
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0545805..a51443c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1277,7 +1277,6 @@
datanode unless the datanode confirms the completion.
</description>
</property>
-
<property>
<name>hdds.db.profile</name>
<value>DISK</value>
@@ -1286,7 +1285,6 @@
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
</property>
-
<property>
<name>hdds.datanode.replication.work.dir</name>
<tag>DATANODE</tag>
@@ -1389,5 +1387,29 @@
</description>
</property>
+ <property>
+ <name>ozone.scm.kerberos.keytab.file</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description> The keytab file used by each SCM daemon to login as its
+ service principal. The principal name is configured with
+ ozone.scm.kerberos.principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.kerberos.principal</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+ </property>
+
+ <property>
+ <name>ozone.scm.web.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ </property>
+ <property>
+ <name>ozone.scm.web.authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/HTTP.keytab</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index e3b5370..8049e9d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -36,11 +36,15 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
import java.io.IOException;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
/**
* The protocol spoken between datanodes and SCM. For specifics please the
* Protoc file that defines this protocol.
*/
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerDatanodeProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9b28b5a..9c32ef8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -19,7 +19,10 @@ package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos
.StorageContainerDatanodeProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from a datanode to StorageContainerManager. This extends
@@ -29,6 +32,9 @@ import org.apache.hadoop.ipc.ProtocolInfo;
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerDatanodeProtocolPB extends
StorageContainerDatanodeProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2d27984..519f51b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -28,10 +28,12 @@ import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
@@ -82,6 +84,9 @@ import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.common.StorageInfo;
import org.apache.hadoop.ozone.lease.LeaseManager;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
@@ -101,6 +106,10 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -189,10 +198,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
*
* @param conf configuration
*/
- private StorageContainerManager(OzoneConfiguration conf) throws IOException {
+ private StorageContainerManager(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
StorageContainerManager.initMetrics();
initContainerReportCache(conf);
+ // Authenticate SCM if security is enabled
+ if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+ OZONE_SECURITY_ENABLED_DEFAULT)) {
+ loginAsSCMUser(conf);
+ }
scmStorage = new SCMStorage(conf);
if (scmStorage.getState() != StorageState.INITIALIZED) {
@@ -310,6 +325,33 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
/**
+ * Login as the configured user for SCM.
+ *
+ * @param conf
+ */
+ private void loginAsSCMUser(Configuration conf)
+ throws IOException, AuthenticationException {
+ LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
+ + "Principal: {}, keytab: {}", conf.get
+ (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+
+ if (SecurityUtil.getAuthenticationMethod(conf).equals
+ (AuthenticationMethod.KERBEROS)) {
+ UserGroupInformation.setConfiguration(conf);
+ InetSocketAddress socAddr = HddsServerUtil
+ .getScmBlockClientBindAddress(conf);
+ SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ } else {
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+ conf) + " authentication method not support. "
+ + "SCM user login failed.");
+ }
+ LOG.info("SCM login successful.");
+ }
+
+ /**
* Builds a message for logging startup information about an RPC server.
*
* @param description RPC server description
@@ -403,8 +445,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @throws IOException
*/
@VisibleForTesting
- public static StorageContainerManager createSCM(
- String[] args, OzoneConfiguration conf) throws IOException {
+ public static StorageContainerManager createSCM(String[] args,
+ OzoneConfiguration conf) throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -420,7 +462,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private static StorageContainerManager createSCM(
String[] args,
OzoneConfiguration conf,
- boolean printBanner) throws IOException {
+ boolean printBanner)
+ throws IOException, AuthenticationException {
String[] argv = (args == null) ? new String[0] : args;
if (!HddsUtils.isHddsEnabled(conf)) {
System.err.println(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 75b2036..da936ad 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.server;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.BaseHttpServer;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import java.io.IOException;
@@ -63,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.OZONE_SCM_KEYTAB_FILE;
+ return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+ return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index ce7fa2f..71a0144 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -113,7 +114,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
}
private static StorageContainerManager getScm(OzoneConfiguration conf)
- throws IOException {
+ throws IOException, AuthenticationException {
conf.setBoolean(OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
if(scmStore.getState() != StorageState.INITIALIZED) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index be5670d..f5b2cd9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.client.protocol;
import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* An implementer of this interface is capable of connecting to Ozone Cluster
@@ -38,6 +40,7 @@ import java.util.List;
* includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
* {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index 4c022fb..a05b9ae 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -70,10 +70,18 @@ nameStartOpt="$nameStartOpt $*"
SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED}
+# == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
+
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
#---------------------------------------------------------
# Check if ozone is enabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index fe902eb..c07d42b 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,13 +47,12 @@ else
exit 1
fi
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
#---------------------------------------------------------
# Check if ozone is enabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index 67a166c..1867d95 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -40,6 +40,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-objectstore-service</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
index 15ee6f6..0573b38 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -107,8 +108,8 @@ public class TestContainerStateManagerIntegration {
}
@Test
- public void testContainerStateManagerRestart()
- throws IOException, TimeoutException, InterruptedException {
+ public void testContainerStateManagerRestart() throws IOException,
+ TimeoutException, InterruptedException, AuthenticationException {
// Allocate 5 containers in ALLOCATED state and 5 in CREATING state
for (int i = 0; i < 10; i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index b352e36..3fe5b76 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import java.io.IOException;
@@ -149,8 +150,7 @@ public interface MiniOzoneCluster {
* @throws TimeoutException
* @throws InterruptedException
*/
- void restartStorageContainerManager() throws InterruptedException,
- TimeoutException, IOException;
+ void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..7b0d73d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.protocolPB
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.Logger;
@@ -232,8 +233,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
- public void restartStorageContainerManager()
- throws TimeoutException, InterruptedException, IOException {
+ public void restartStorageContainerManager() throws TimeoutException,
+ InterruptedException, IOException, AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
@@ -369,9 +370,16 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
public MiniOzoneCluster build() throws IOException {
DefaultMetricsSystem.setMiniClusterMode(true);
initializeConfiguration();
- StorageContainerManager scm = createSCM();
- scm.start();
- OzoneManager om = createOM();
+ StorageContainerManager scm;
+ OzoneManager om;
+ try {
+ scm = createSCM();
+ scm.start();
+ om = createOM();
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
om.start();
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
@@ -423,7 +431,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
*
* @throws IOException
*/
- private StorageContainerManager createSCM() throws IOException {
+ private StorageContainerManager createSCM()
+ throws IOException, AuthenticationException {
configureSCM();
SCMStorage scmStore = new SCMStorage(conf);
initializeScmStorage(scmStore);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
new file mode 100644
index 0000000..9c430ad
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+import java.util.UUID;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.server.SCMStorage;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class to for security enabled Ozone cluster.
+ */
+@InterfaceAudience.Private
+public final class TestSecureOzoneCluster {
+
+ private Logger LOGGER = LoggerFactory
+ .getLogger(TestSecureOzoneCluster.class);
+
+ private MiniKdc miniKdc;
+ private OzoneConfiguration conf;
+ private File workDir;
+ private static Properties securityProperties;
+ private File scmKeytab;
+ private File spnegoKeytab;
+ private String curUser;
+
+ @Before
+ public void init() {
+ try {
+ conf = new OzoneConfiguration();
+ startMiniKdc();
+ setSecureConfig(conf);
+ createCredentialsInKDC(conf, miniKdc);
+ } catch (IOException e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ } catch (Exception e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ }
+ }
+
+ private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
+ throws Exception {
+ createPrincipal(scmKeytab,
+ conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ }
+
+ private void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private void startMiniKdc() throws Exception {
+ workDir = GenericTestUtils
+ .getTestDir(TestSecureOzoneCluster.class.getSimpleName());
+ securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private void setSecureConfig(Configuration conf) throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ String host = KerberosUtil.getLocalHostName();
+ String realm = miniKdc.getRealm();
+ curUser = UserGroupInformation.getCurrentUser()
+ .getUserName();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/" + host + "@" + realm);
+ conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_SCM/" + host + "@" + realm);
+
+ scmKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ scmKeytab.getAbsolutePath());
+ conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ spnegoKeytab.getAbsolutePath());
+
+ }
+
+ @Test
+ public void testSecureScmStartupSuccess() throws Exception {
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path scmPath = Paths.get(path, "scm-meta");
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+ SCMStorage scmStore = new SCMStorage(conf);
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ scmStore.setClusterId(clusterId);
+ scmStore.setScmId(scmId);
+ // writes the version file properties
+ scmStore.initialize();
+ StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
+ //Reads the SCM Info from SCM instance
+ ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+ Assert.assertEquals(clusterId, scmInfo.getClusterId());
+ Assert.assertEquals(scmId, scmInfo.getScmId());
+ }
+
+ @Test
+ public void testSecureScmStartupFailure() throws Exception {
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path scmPath = Paths.get(path, "scm-meta");
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm@" + miniKdc.getRealm());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+
+ SCMStorage scmStore = new SCMStorage(conf);
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ scmStore.setClusterId(clusterId);
+ scmStore.setScmId(scmId);
+ // writes the version file properties
+ scmStore.initialize();
+ LambdaTestUtils.intercept(IOException.class,
+ "Running in secure mode, but config doesn't have a keytab",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/_HOST@EXAMPLE.com");
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ "/etc/security/keytabs/scm.keytab");
+
+ LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
+ + "to login: for principal:",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "OAuth2");
+
+ LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ + " attribute value for hadoop.security.authentication of OAuth2",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "KERBEROS_SSL");
+ LambdaTestUtils.intercept(AuthenticationException.class,
+ "KERBEROS_SSL authentication method not support.",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa7ca153/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 23d371d..c79da17 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -59,8 +59,10 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
+
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -426,7 +428,8 @@ public class TestStorageContainerManager {
}
@Test
- public void testSCMInitializationFailure() throws IOException {
+ public void testSCMInitializationFailure()
+ throws IOException, AuthenticationException {
OzoneConfiguration conf = new OzoneConfiguration();
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
@@ -439,7 +442,8 @@ public class TestStorageContainerManager {
}
@Test
- public void testSCMInitializationReturnCode() throws IOException {
+ public void testSCMInitializationReturnCode() throws IOException,
+ AuthenticationException {
ExitUtil.disableSystemExit();
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-7. Enable kerberos auth for
Ozone client in hadoop rpc. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-7. Enable kerberos auth for Ozone client in hadoop rpc. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c821937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c821937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c821937
Branch: refs/heads/HDDS-4
Commit: 5c821937a4972c9a6001c18e18b7a58891198011
Parents: 074b8f4
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri May 18 13:09:17 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800
----------------------------------------------------------------------
.../src/test/compose/compose-secure/.env | 17 ++++
.../compose/compose-secure/docker-compose.yaml | 66 ++++++++++++++
.../test/compose/compose-secure/docker-config | 66 ++++++++++++++
.../acceptance/ozone-secure.robot | 95 ++++++++++++++++++++
.../hadoop/ozone/client/rest/RestClient.java | 4 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 6 +-
6 files changed, 248 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
new file mode 100644
index 0000000..3254735
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
new file mode 100644
index 0000000..2661163
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ ozone.kdc:
+ image: ahadoop/kdc:v1
+ namenode:
+ image: ahadoop/ozone:v1
+ hostname: namenode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9000:9000
+ environment:
+ ENSURE_NAMENODE_DIR: /data/namenode
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","namenode"]
+ datanode:
+ image: ahadoop/ozone:v1
+ hostname: datanode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ ksm:
+ image: ahadoop/ozone:v1
+ hostname: ksm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","ksm"]
+ scm:
+ image: ahadoop/ozone:v1
+ hostname: scm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
new file mode 100644
index 0000000..678c75a
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_hdds.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_hdds.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+HDFS-SITE.XML_dfs.block.access.token.enable=true
+HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=ozone.kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=10
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
new file mode 100644
index 0000000..4a78980
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test to start cluster with docker-compose environments.
+Library OperatingSystem
+Suite Setup Startup Ozone Cluster
+Suite Teardown Teardown Ozone Cluster
+
+*** Variables ***
+${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
+${version}
+
+*** Test Cases ***
+
+Daemons are running
+ Is daemon running ksm
+ Is daemon running scm
+ Is daemon running datanode
+ Is daemon running ozone.kdc
+
+Check if datanode is connected to the scm
+ Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
+
+Test rest interface
+ ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+ Should contain ${result} 201 Created
+ ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+ Should contain ${result} 201 Created
+ ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+ Should contain ${result} 200 OK
+ ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+ Should contain ${result} 200 OK
+
+Test ozone cli
+ ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ Should contain ${result} Client cannot authenticate via
+ # Authenticate testuser
+ Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+ Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+ Should contain ${result} createdOn
+ Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
+ ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+ Should Be Equal ${result} bill
+
+*** Keywords ***
+
+Startup Ozone Cluster
+ ${rc} ${output} = Run docker compose 0 down
+ ${rc} ${output} = Run docker compose 0 up -d
+ Should Be Equal As Integers ${rc} 0
+ Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening
+
+Teardown Ozone Cluster
+ Run docker compose 0 down
+
+Is daemon running
+ [arguments] ${name}
+ ${result} = Run docker ps
+ Should contain ${result} _${name}_1
+
+Is Daemon started
+ [arguments] ${name} ${expression}
+ ${rc} ${result} = Run docker compose 0 logs
+ Should contain ${result} ${expression}
+
+Have healthy datanodes
+ [arguments] ${requirednodes}
+ ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
+ Should Be Equal ${result} ${requirednodes}
+
+Execute on
+ [arguments] ${expected_rc} ${componentname} ${command}
+ ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command}
+ [return] ${return}
+
+Run docker compose
+ [arguments] ${expected_rc} ${command}
+ Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone
+ ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
+ Should Be Equal As Integers ${rc} ${expected_rc}
+ [return] ${rc} ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 8889d99..3fd0927 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -111,7 +111,7 @@ public class RestClient implements ClientProtocol {
try {
Preconditions.checkNotNull(conf);
this.conf = conf;
-
+ this.ugi = UserGroupInformation.getCurrentUser();
long socketTimeout = conf.getTimeDuration(
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
@@ -147,7 +147,7 @@ public class RestClient implements ClientProtocol {
.setConnectTimeout(Math.toIntExact(connectionTimeout))
.build())
.build();
- this.ugi = UserGroupInformation.getCurrentUser();
+
this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c821937/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 65adbfa..1b70f3b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -126,8 +126,7 @@ public class RpcClient implements ClientProtocol {
this.ozoneManagerClient =
new OzoneManagerProtocolClientSideTranslatorPB(
RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
- omAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
+ omAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
long scmVersion =
@@ -138,8 +137,7 @@ public class RpcClient implements ClientProtocol {
this.storageContainerLocationClient =
new StorageContainerLocationProtocolClientSideTranslatorPB(
RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
+ scmAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
this.xceiverClientManager = new XceiverClientManager(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDDS-684. Fix HDDS-4 branch after
HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-684. Fix HDDS-4 branch after HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72468986
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72468986
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72468986
Branch: refs/heads/HDDS-4
Commit: 724689864b6723a2ada9ccb925f9b828c19f305b
Parents: 950a473
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 24 15:46:34 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
hadoop-ozone/dist/dev-support/bin/dist-layout-stitching | 2 +-
.../dist/src/main/compose/ozonesecure/docker-config | 1 +
.../ozonesecure/docker-image/runner/scripts/starter.sh | 4 ++--
.../java/org/apache/hadoop/ozone/om/OzoneManager.java | 12 ++++++------
4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index e736065..2a4f0ad 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -111,5 +111,5 @@ run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore
cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./
#Copy docker compose files
-run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" .
+run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/compose" .
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index 704dc7b..36f05ae 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -37,6 +37,7 @@ HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
index 04cd49d..eec7ce9 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -82,7 +82,7 @@ fi
if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
- /opt/hadoop/bin/ozone scm -init
+ /opt/hadoop/bin/ozone scm --init
fi
fi
@@ -92,7 +92,7 @@ if [ -n "$ENSURE_OM_INITIALIZED" ]; then
# Could be removed after HDFS-13203
echo "Waiting 15 seconds for SCM startup"
sleep 15
- /opt/hadoop/bin/ozone om -createObjectStore
+ /opt/hadoop/bin/ozone om --init
fi
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72468986/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 7fc6aee..a5cec20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -259,17 +259,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
- * Login KSM service user if security and Kerberos are enabled.
+ * Login OM service user if security and Kerberos are enabled.
*
* @param conf
* @throws IOException, AuthenticationException
*/
- private static void loginKSMUser(OzoneConfiguration conf)
+ private static void loginOMUser(OzoneConfiguration conf)
throws IOException, AuthenticationException {
if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) {
- LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ LOG.debug("Ozone security is enabled. Attempting login for OM user. "
+ "Principal: {},keytab: {}", conf.get(
OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
@@ -281,10 +281,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
- (conf) + " authentication method not supported. KSM user login "
+ (conf) + " authentication method not supported. OM user login "
+ "failed.");
}
- LOG.info("KSM login successful.");
+ LOG.info("Ozone Manager login successful.");
}
/**
@@ -440,7 +440,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
// Authenticate KSM if security is enabled
if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
- loginKSMUser(conf);
+ loginOMUser(conf);
}
switch (startOpt) {
case INIT:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: Revert "Bad merge with
996a627b289947af3894bf83e7b63ec702a665cd"
Posted by xy...@apache.org.
Revert "Bad merge with 996a627b289947af3894bf83e7b63ec702a665cd"
This reverts commit 996a627b289947af3894bf83e7b63ec702a665cd.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/074b8f4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/074b8f4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/074b8f4e
Branch: refs/heads/HDDS-4
Commit: 074b8f4ed52a9defaedaff788c517d7b3f3dd024
Parents: fcd705a
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 15 16:56:24 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/074b8f4e/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4d79e8c..d3e352b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -198,7 +198,6 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -256,9 +255,6 @@
</property>
<property>
<name>hdds.container.report.interval</name>
-=======
- <name>ozone.container.report.interval</name>
->>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
<value>60000ms</value>
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>Time interval of the datanode to send container report. Each
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDFS-13713. Add specification of
Multipart Upload API to FS specification, with contract tests.
Posted by xy...@apache.org.
HDFS-13713. Add specification of Multipart Upload API to FS specification, with contract tests.
Contributed by Ewan Higgs and Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1d24f84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1d24f84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1d24f84
Branch: refs/heads/HDDS-4
Commit: c1d24f848345f6d34a2ac2d570d49e9787a0df6a
Parents: b71cc7f
Author: Ewan Higgs <eh...@apache.org>
Authored: Thu Nov 29 15:11:07 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 15:12:17 2018 +0000
----------------------------------------------------------------------
.../hadoop/fs/FileSystemMultipartUploader.java | 36 +-
.../org/apache/hadoop/fs/MultipartUploader.java | 88 ++-
.../hadoop/fs/MultipartUploaderFactory.java | 7 +
.../src/site/markdown/filesystem/index.md | 1 +
.../markdown/filesystem/multipartuploader.md | 235 ++++++++
.../AbstractContractMultipartUploaderTest.java | 565 +++++++++++++++----
.../TestLocalFSContractMultipartUploader.java | 10 +
.../hdfs/TestHDFSContractMultipartUploader.java | 15 +
.../hadoop/fs/s3a/S3AMultipartUploader.java | 31 +-
.../s3a/ITestS3AContractMultipartUploader.java | 64 ++-
.../fs/s3a/TestS3AMultipartUploaderSupport.java | 2 +-
11 files changed, 876 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 94c7861..b77c244 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -19,21 +19,23 @@ package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.stream.Collectors;
import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
import org.apache.commons.compress.utils.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.Path.mergePaths;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* A MultipartUploader that uses the basic FileSystem commands.
@@ -70,7 +72,8 @@ public class FileSystemMultipartUploader extends MultipartUploader {
public PartHandle putPart(Path filePath, InputStream inputStream,
int partNumber, UploadHandle uploadId, long lengthInBytes)
throws IOException {
-
+ checkPutArguments(filePath, inputStream, partNumber, uploadId,
+ lengthInBytes);
byte[] uploadIdByteArray = uploadId.toByteArray();
checkUploadId(uploadIdByteArray);
Path collectorPath = new Path(new String(uploadIdByteArray, 0,
@@ -82,16 +85,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
fs.createFile(partPath).build()) {
IOUtils.copy(inputStream, fsDataOutputStream, 4096);
} finally {
- org.apache.hadoop.io.IOUtils.cleanupWithLogger(LOG, inputStream);
+ cleanupWithLogger(LOG, inputStream);
}
return BBPartHandle.from(ByteBuffer.wrap(
partPath.toString().getBytes(Charsets.UTF_8)));
}
private Path createCollectorPath(Path filePath) {
+ String uuid = UUID.randomUUID().toString();
return mergePaths(filePath.getParent(),
mergePaths(new Path(filePath.getName().split("\\.")[0]),
- mergePaths(new Path("_multipart"),
+ mergePaths(new Path("_multipart_" + uuid),
new Path(Path.SEPARATOR))));
}
@@ -110,21 +114,16 @@ public class FileSystemMultipartUploader extends MultipartUploader {
@Override
@SuppressWarnings("deprecation") // rename w/ OVERWRITE
- public PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
- throws IOException {
+ public PathHandle complete(Path filePath, Map<Integer, PartHandle> handleMap,
+ UploadHandle multipartUploadId) throws IOException {
checkUploadId(multipartUploadId.toByteArray());
- if (handles.isEmpty()) {
- throw new IOException("Empty upload");
- }
- // If destination already exists, we believe we already completed it.
- if (fs.exists(filePath)) {
- return getPathHandle(filePath);
- }
+ checkPartHandles(handleMap);
+ List<Map.Entry<Integer, PartHandle>> handles =
+ new ArrayList<>(handleMap.entrySet());
+ handles.sort(Comparator.comparingInt(Map.Entry::getKey));
- handles.sort(Comparator.comparing(Pair::getKey));
List<Path> partHandles = handles
.stream()
.map(pair -> {
@@ -134,7 +133,10 @@ public class FileSystemMultipartUploader extends MultipartUploader {
})
.collect(Collectors.toList());
- Path collectorPath = createCollectorPath(filePath);
+ byte[] uploadIdByteArray = multipartUploadId.toByteArray();
+ Path collectorPath = new Path(new String(uploadIdByteArray, 0,
+ uploadIdByteArray.length, Charsets.UTF_8));
+
boolean emptyFile = totalPartsLen(partHandles) == 0;
if (emptyFile) {
fs.create(filePath).close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
index b56dbf8..7ed987e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
@@ -17,40 +17,45 @@
*/
package org.apache.hadoop.fs;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
-import java.util.List;
+import java.util.Map;
-import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.tuple.Pair;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import static com.google.common.base.Preconditions.checkArgument;
+
/**
* MultipartUploader is an interface for copying files multipart and across
* multiple nodes. Users should:
* <ol>
- * <li>Initialize an upload</li>
- * <li>Upload parts in any order</li>
+ * <li>Initialize an upload.</li>
+ * <li>Upload parts in any order.</li>
* <li>Complete the upload in order to have it materialize in the destination
- * FS</li>
+ * FS.</li>
* </ol>
- *
- * Implementers should make sure that the complete function should make sure
- * that 'complete' will reorder parts if the destination FS doesn't already
- * do it for them.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public abstract class MultipartUploader {
+public abstract class MultipartUploader implements Closeable {
public static final Logger LOG =
LoggerFactory.getLogger(MultipartUploader.class);
/**
+ * Perform any cleanup.
+ * The upload is not required to support any operations after this.
+ * @throws IOException problems on close.
+ */
+ @Override
+ public void close() throws IOException {
+ }
+
+ /**
* Initialize a multipart upload.
* @param filePath Target path for upload.
* @return unique identifier associating part uploads.
@@ -59,8 +64,8 @@ public abstract class MultipartUploader {
public abstract UploadHandle initialize(Path filePath) throws IOException;
/**
- * Put part as part of a multipart upload. It should be possible to have
- * parts uploaded in any order (or in parallel).
+ * Put part as part of a multipart upload.
+ * It is possible to have parts uploaded in any order (or in parallel).
* @param filePath Target path for upload (same as {@link #initialize(Path)}).
* @param inputStream Data for this part. Implementations MUST close this
* stream after reading in the data.
@@ -77,15 +82,15 @@ public abstract class MultipartUploader {
/**
* Complete a multipart upload.
* @param filePath Target path for upload (same as {@link #initialize(Path)}.
- * @param handles non-empty list of identifiers with associated part numbers
+ * @param handles non-empty map of part number to part handle.
* from {@link #putPart(Path, InputStream, int, UploadHandle, long)}.
- * Depending on the backend, the list order may be significant.
* @param multipartUploadId Identifier from {@link #initialize(Path)}.
* @return unique PathHandle identifier for the uploaded file.
- * @throws IOException IO failure or the handle list is empty.
+ * @throws IOException IO failure
*/
public abstract PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
+ Map<Integer, PartHandle> handles,
+ UploadHandle multipartUploadId)
throws IOException;
/**
@@ -98,13 +103,52 @@ public abstract class MultipartUploader {
throws IOException;
/**
- * Utility method to validate uploadIDs
- * @param uploadId
- * @throws IllegalArgumentException
+ * Utility method to validate uploadIDs.
+ * @param uploadId Upload ID
+ * @throws IllegalArgumentException invalid ID
*/
protected void checkUploadId(byte[] uploadId)
throws IllegalArgumentException {
- Preconditions.checkArgument(uploadId.length > 0,
+ checkArgument(uploadId != null, "null uploadId");
+ checkArgument(uploadId.length > 0,
"Empty UploadId is not valid");
}
+
+ /**
+ * Utility method to validate partHandles.
+ * @param partHandles handles
+ * @throws IllegalArgumentException if the parts are invalid
+ */
+ protected void checkPartHandles(Map<Integer, PartHandle> partHandles) {
+ checkArgument(!partHandles.isEmpty(),
+ "Empty upload");
+ partHandles.keySet()
+ .stream()
+ .forEach(key ->
+ checkArgument(key > 0,
+ "Invalid part handle index %s", key));
+ }
+
+ /**
+ * Check all the arguments to the
+ * {@link #putPart(Path, InputStream, int, UploadHandle, long)} operation.
+ * @param filePath Target path for upload (same as {@link #initialize(Path)}).
+ * @param inputStream Data for this part. Implementations MUST close this
+ * stream after reading in the data.
+ * @param partNumber Index of the part relative to others.
+ * @param uploadId Identifier from {@link #initialize(Path)}.
+ * @param lengthInBytes Target length to read from the stream.
+ * @throws IllegalArgumentException invalid argument
+ */
+ protected void checkPutArguments(Path filePath,
+ InputStream inputStream,
+ int partNumber,
+ UploadHandle uploadId,
+ long lengthInBytes) throws IllegalArgumentException {
+ checkArgument(filePath != null, "null filePath");
+ checkArgument(inputStream != null, "null inputStream");
+ checkArgument(partNumber > 0, "Invalid part number: %d", partNumber);
+ checkArgument(uploadId != null, "null uploadId");
+ checkArgument(lengthInBytes >= 0, "Invalid part length: %d", lengthInBytes);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
index 8b35232..e35b6bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
@@ -52,6 +52,13 @@ public abstract class MultipartUploaderFactory {
}
}
+ /**
+ * Get the multipart loader for a specific filesystem.
+ * @param fs filesystem
+ * @param conf configuration
+ * @return an uploader, or null if one was found.
+ * @throws IOException failure during the creation process.
+ */
public static MultipartUploader get(FileSystem fs, Configuration conf)
throws IOException {
MultipartUploader mpu = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
index 532b6c7..6b4399e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
@@ -36,3 +36,4 @@ HDFS as these are commonly expected by Hadoop client applications.
1. [FSDataOutputStreamBuilder class](fsdataoutputstreambuilder.html)
2. [Testing with the Filesystem specification](testing.html)
2. [Extending the specification and its tests](extending.html)
+1. [Uploading a file using Multiple Parts](multipartuploader.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
new file mode 100644
index 0000000..629c0c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
@@ -0,0 +1,235 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+<!-- ============================================================= -->
+<!-- CLASS: MultipartUploader -->
+<!-- ============================================================= -->
+
+# class `org.apache.hadoop.fs.MultipartUploader`
+
+<!-- MACRO{toc|fromDepth=1|toDepth=2} -->
+
+The abstract `MultipartUploader` class is the original class to upload a file
+using multiple parts to Hadoop-supported filesystems. The benefits of a
+multipart upload is that the file can be uploaded from multiple clients or
+processes in parallel and the results will not be visible to other clients until
+the `complete` function is called.
+
+When implemented by an object store, uploaded data may incur storage charges,
+even before it is visible in the filesystems. Users of this API must be diligent
+and always perform best-effort attempts to complete or abort the upload.
+
+## Invariants
+
+All the requirements of a valid MultipartUploader are considered implicit
+econditions and postconditions:
+all operations on a valid MultipartUploader MUST result in a new
+MultipartUploader that is also valid.
+
+The operations of a single multipart upload may take place across different
+instance of a multipart uploader, across different processes and hosts.
+It is therefore a requirement that:
+
+1. All state needed to upload a part, complete an upload or abort an upload
+must be contained within or retrievable from an upload handle.
+
+1. If an upload handle is marshalled to another process, then, if the
+receiving process has the correct permissions, it may participate in the
+upload, by uploading one or more parts, by completing an upload, and/or by
+aborting the upload.
+
+## Concurrency
+
+Multiple processes may upload parts of a multipart upload simultaneously.
+
+If a call is made to `initialize(path)` to a destination where an active
+upload is in progress, implementations MUST perform one of the two operations.
+
+* Reject the call as a duplicate.
+* Permit both to proceed, with the final output of the file being
+that of _exactly one of the two uploads_.
+
+Which upload succeeds is undefined. Users must not expect consistent
+behavior across filesystems, across filesystem instances *or even
+across different requests.
+
+If a multipart upload is completed or aborted while a part upload is in progress,
+the in-progress upload, if it has not completed, must not be included in
+the final file, in whole or in part. Implementations SHOULD raise an error
+in the `putPart()` operation.
+
+## Model
+
+A File System which supports Multipart Uploads extends the existing model
+`(Directories, Files, Symlinks)` to one of `(Directories, Files, Symlinks, Uploads)`
+`Uploads` of type `Map[UploadHandle -> Map[PartHandle -> UploadPart]`.
+
+
+The `Uploads` element of the state tuple is a map of all active uploads.
+
+```python
+Uploads: Map[UploadHandle -> Map[PartHandle -> UploadPart]`
+```
+
+An UploadHandle is a non-empty list of bytes.
+```python
+UploadHandle: List[byte]
+len(UploadHandle) > 0
+```
+
+Clients *MUST* treat this as opaque. What is core to this features design is that the handle is valid from
+across clients: the handle may be serialized on host `hostA`, deserialized on `hostB` and still used
+to extend or complete the upload.
+
+```python
+UploadPart = (Path: path, parts: Map[PartHandle -> byte[]])
+```
+
+Similarly, the `PartHandle` type is also a non-empty list of opaque bytes, again, marshallable between hosts.
+
+```python
+PartHandle: List[byte]
+```
+
+It is implicit that each `UploadHandle` in `FS.Uploads` is unique.
+Similarly, each `PartHandle` in the map of `[PartHandle -> UploadPart]` must also be unique.
+
+1. There is no requirement that Part Handles are unique across uploads.
+1. There is no requirement that Upload Handles are unique over time.
+However, if Part Handles are rapidly recycled, there is a risk that the nominally
+idempotent operation `abort(FS, uploadHandle)` could unintentionally cancel a
+successor operation which used the same Upload Handle.
+
+## State Changing Operations
+
+### `UploadHandle initialize(Path path)`
+
+Initialized a Multipart Upload, returning an upload handle for use in
+subsequent operations.
+
+#### Preconditions
+
+```python
+if path == "/" : raise IOException
+
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+```
+
+If a filesystem does not support concurrent uploads to a destination,
+then the following precondition is added
+
+```python
+if path in values(FS.Uploads) raise PathExistsException, IOException
+
+```
+
+
+#### Postconditions
+
+The outcome of this operation is that the filesystem state is updated with a new
+active upload, with a new handle, this handle being returned to the caller.
+
+```python
+handle' = UploadHandle where not handle' in keys(FS.Uploads)
+FS' = FS where FS'.Uploads(handle') == {}
+result = handle'
+```
+
+### `PartHandle putPart(Path path, InputStream inputStream, int partNumber, UploadHandle uploadHandle, long lengthInBytes)`
+
+Upload a part for the multipart upload.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads)
+partNumber >= 1
+lengthInBytes >= 0
+len(inputStream) >= lengthInBytes
+```
+
+#### Postconditions
+
+```python
+data' = inputStream(0..lengthInBytes)
+partHandle' = byte[] where not partHandle' in keys(FS.uploads(uploadHandle).parts)
+FS' = FS where FS'.uploads(uploadHandle).parts(partHandle') == data'
+result = partHandle'
+```
+
+The data is stored in the filesystem, pending completion.
+
+
+### `PathHandle complete(Path path, Map<Integer, PartHandle> parts, UploadHandle multipartUploadId)`
+
+Complete the multipart upload.
+
+A Filesystem may enforce a minimum size of each part, excluding the last part uploaded.
+
+If a part is out of this range, an `IOException` MUST be raised.
+
+#### Preconditions
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+FS.Uploads(uploadHandle).path == path
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+parts.size() > 0
+```
+
+If there are handles in the MPU which aren't included in the map, then the omitted
+parts will not be a part of the resulting file. It is up to the implementation
+of the MultipartUploader to make sure the leftover parts are cleaned up.
+
+In the case of backing stores that support directories (local filesystem, HDFS,
+etc), if, at the point of completion, there is now a directory at the
+destination then a `PathIsDirectoryException` or other `IOException` must be thrown.
+
+#### Postconditions
+
+```python
+UploadData' == ordered concatention of all data in the map of parts, ordered by key
+exists(FS', path') and result = PathHandle(path')
+FS' = FS where FS.Files(path) == UploadData' and not uploadHandle in keys(FS'.uploads)
+```
+
+The PathHandle is returned by the complete operation so subsequent operations
+will be able to identify that the data has not changed in the meantime.
+
+The order of parts in the uploaded by file is that of the natural order of
+parts: part 1 is ahead of part 2, etc.
+
+
+### `void abort(Path path, UploadHandle multipartUploadId)`
+
+Abort a multipart upload. The handle becomes invalid and not subject to reuse.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+```
+
+#### Postconditions
+
+The upload handle is no longer known.
+
+```python
+FS' = FS where not uploadHandle in keys(FS'.uploads)
+```
+A subsequent call to `abort()` with the same handle will fail, unless
+the handle has been recycled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 7cee5a6..7a8f083 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -23,15 +23,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
import com.google.common.base.Charsets;
+import org.junit.Assume;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BBUploadHandle;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -43,11 +47,64 @@ import org.apache.hadoop.fs.PathHandle;
import org.apache.hadoop.fs.UploadHandle;
import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+import static org.apache.hadoop.test.LambdaTestUtils.eventually;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public abstract class AbstractContractMultipartUploaderTest extends
AbstractFSContractTestBase {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
+ /**
+ * Size of very small uploads.
+ * Enough to be non empty, not big enough to cause delays on uploads.
+ */
+ protected static final int SMALL_FILE = 100;
+
+ private MultipartUploader mpu;
+ private MultipartUploader mpu2;
+ private final Random random = new Random();
+ private UploadHandle activeUpload;
+ private Path activeUploadPath;
+
+ protected String getMethodName() {
+ return methodName.getMethodName();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ Configuration conf = getContract().getConf();
+ mpu = MultipartUploaderFactory.get(getFileSystem(), conf);
+ mpu2 = MultipartUploaderFactory.get(getFileSystem(), conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ if (mpu!= null && activeUpload != null) {
+ try {
+ mpu.abort(activeUploadPath, activeUpload);
+ } catch (FileNotFoundException ignored) {
+ /* this is fine */
+ } catch (Exception e) {
+ LOG.info("in teardown", e);
+ }
+ }
+ cleanupWithLogger(LOG, mpu, mpu2);
+ super.teardown();
+ }
+
+ /**
+ * Get a test path based on the method name.
+ * @return a path to use in the test
+ * @throws IOException failure to build the path name up.
+ */
+ protected Path methodPath() throws IOException {
+ return path(getMethodName());
+ }
+
/**
* The payload is the part number repeated for the length of the part.
* This makes checking the correctness of the upload straightforward.
@@ -55,9 +112,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
* @return the bytes to upload.
*/
private byte[] generatePayload(int partNumber) {
- int sizeInBytes = partSizeInBytes();
- ByteBuffer buffer = ByteBuffer.allocate(sizeInBytes);
- for (int i=0 ; i < sizeInBytes/(Integer.SIZE/Byte.SIZE); ++i) {
+ return generatePayload(partNumber, partSizeInBytes());
+ }
+
+ /**
+ * Generate a payload of a given size; part number is used
+ * for all the values.
+ * @param partNumber part number
+ * @param size size in bytes
+ * @return the bytes to upload.
+ */
+ private byte[] generatePayload(final int partNumber, final int size) {
+ ByteBuffer buffer = ByteBuffer.allocate(size);
+ for (int i=0; i < size /(Integer.SIZE/Byte.SIZE); ++i) {
buffer.putInt(partNumber);
}
return buffer.array();
@@ -70,11 +137,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
* @throws IOException failure to read or digest the file.
*/
protected byte[] digest(Path path) throws IOException {
- FileSystem fs = getFileSystem();
- try (InputStream in = fs.open(path)) {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ try (InputStream in = getFileSystem().open(path)) {
byte[] fdData = IOUtils.toByteArray(in);
MessageDigest newDigest = DigestUtils.getMd5Digest();
- return newDigest.digest(fdData);
+ byte[] digest = newDigest.digest(fdData);
+ return digest;
+ } finally {
+ timer.end("Download and digest of path %s", path);
}
}
@@ -93,74 +163,230 @@ public abstract class AbstractContractMultipartUploaderTest extends
}
/**
+ * How long in milliseconds for propagation of
+ * store changes, including update/delete/list
+ * to be everywhere.
+ * If 0: the FS is consistent.
+ * @return a time in milliseconds.
+ */
+ protected int timeToBecomeConsistentMillis() {
+ return 0;
+ }
+
+ /**
+ * Does a call to finalize an upload (either complete or abort) consume the
+ * uploadID immediately or is it reaped at a later point in time?
+ * @return true if the uploadID will be consumed immediately (and no longer
+ * resuable).
+ */
+ protected abstract boolean finalizeConsumesUploadIdImmediately();
+
+ /**
+ * Does the store support concurrent uploads to the same destination path?
+ * @return true if concurrent uploads are supported.
+ */
+ protected abstract boolean supportsConcurrentUploadsToSamePath();
+
+ /**
+ * Pick a multipart uploader from the index value.
+ * @param index index of upload
+ * @return an uploader
+ */
+ protected MultipartUploader mpu(int index) {
+ return (index % 2 == 0) ? mpu : mpu2;
+ }
+
+ /**
+ * Pick a multipart uploader at random.
+ * @return an uploader
+ */
+ protected MultipartUploader randomMpu() {
+ return mpu(random.nextInt(10));
+ }
+
+ /**
* Assert that a multipart upload is successful.
* @throws Exception failure
*/
@Test
public void testSingleUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testSingleUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
- byte[] payload = generatePayload(1);
+ int size = SMALL_FILE;
+ byte[] payload = generatePayload(1, size);
origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, 1, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(1, partHandle));
- PathHandle fd = completeUpload(file, mpu, uploadHandle, partHandles,
+ PartHandle partHandle = putPart(file, uploadHandle, 1, payload);
+ partHandles.put(1, partHandle);
+ PathHandle fd = completeUpload(file, uploadHandle, partHandles,
origDigest,
- payload.length);
+ size);
+
+ if (finalizeConsumesUploadIdImmediately()) {
+ intercept(FileNotFoundException.class,
+ () -> mpu.complete(file, partHandles, uploadHandle));
+ } else {
+ PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
+ assertArrayEquals("Path handles differ", fd.toByteArray(),
+ fd2.toByteArray());
+ }
+ }
+
+ /**
+ * Initialize an upload.
+ * This saves the path and upload handle as the active
+ * upload, for aborting in teardown
+ * @param dest destination
+ * @return the handle
+ * @throws IOException failure to initialize
+ */
+ protected UploadHandle initializeUpload(final Path dest) throws IOException {
+ activeUploadPath = dest;
+ activeUpload = randomMpu().initialize(dest);
+ return activeUpload;
+ }
- // Complete is idempotent
- PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
- assertArrayEquals("Path handles differ", fd.toByteArray(),
- fd2.toByteArray());
+ /**
+ * Generate then upload a part.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param index index of part
+ * @param origDigest digest to build up. May be null
+ * @return the part handle
+ * @throws IOException IO failure.
+ */
+ protected PartHandle buildAndPutPart(
+ final Path file,
+ final UploadHandle uploadHandle,
+ final int index,
+ final MessageDigest origDigest) throws IOException {
+ byte[] payload = generatePayload(index);
+ if (origDigest != null) {
+ origDigest.update(payload);
+ }
+ return putPart(file, uploadHandle, index, payload);
}
+ /**
+ * Put a part.
+ * The entire byte array is uploaded.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param index index of part
+ * @param payload byte array of payload
+ * @return the part handle
+ * @throws IOException IO failure.
+ */
+ protected PartHandle putPart(final Path file,
+ final UploadHandle uploadHandle,
+ final int index,
+ final byte[] payload) throws IOException {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ PartHandle partHandle = mpu(index)
+ .putPart(file,
+ new ByteArrayInputStream(payload),
+ index,
+ uploadHandle,
+ payload.length);
+ timer.end("Uploaded part %s", index);
+ LOG.info("Upload bandwidth {} MB/s",
+ timer.bandwidthDescription(payload.length));
+ return partHandle;
+ }
+
+ /**
+ * Complete an upload with the active MPU instance.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param partHandles map of handles
+ * @param origDigest digest of source data (may be null)
+ * @param expectedLength expected length of result.
+ * @return the path handle from the upload.
+ * @throws IOException IO failure
+ */
private PathHandle completeUpload(final Path file,
- final MultipartUploader mpu,
final UploadHandle uploadHandle,
- final List<Pair<Integer, PartHandle>> partHandles,
+ final Map<Integer, PartHandle> partHandles,
final MessageDigest origDigest,
final int expectedLength) throws IOException {
- PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+ PathHandle fd = complete(file, uploadHandle, partHandles);
FileStatus status = verifyPathExists(getFileSystem(),
"Completed file", file);
assertEquals("length of " + status,
expectedLength, status.getLen());
+ if (origDigest != null) {
+ verifyContents(file, origDigest, expectedLength);
+ }
+ return fd;
+ }
+
+ /**
+ * Verify the contents of a file.
+ * @param file path
+ * @param origDigest digest
+ * @param expectedLength expected length (for logging B/W)
+ * @throws IOException IO failure
+ */
+ protected void verifyContents(final Path file,
+ final MessageDigest origDigest,
+ final int expectedLength) throws IOException {
+ ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
assertArrayEquals("digest of source and " + file
+ " differ",
origDigest.digest(), digest(file));
+ timer2.end("Completed digest", file);
+ LOG.info("Download bandwidth {} MB/s",
+ timer2.bandwidthDescription(expectedLength));
+ }
+
+ /**
+ * Perform the inner complete without verification.
+ * @param file destination path
+ * @param uploadHandle upload handle
+ * @param partHandles map of parts
+ * @return the path handle from the upload.
+ * @throws IOException IO failure
+ */
+ private PathHandle complete(final Path file,
+ final UploadHandle uploadHandle,
+ final Map<Integer, PartHandle> partHandles) throws IOException {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ PathHandle fd = randomMpu().complete(file, partHandles, uploadHandle);
+ timer.end("Completed upload to %s", file);
return fd;
}
/**
+ * Abort an upload.
+ * @param file path
+ * @param uploadHandle handle
+ * @throws IOException failure
+ */
+ private void abortUpload(final Path file, UploadHandle uploadHandle)
+ throws IOException {
+ randomMpu().abort(file, uploadHandle);
+ }
+
+ /**
* Assert that a multipart upload is successful.
* @throws Exception failure
*/
@Test
public void testMultipartUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
final int payloadCount = getTestPayloadCount();
for (int i = 1; i <= payloadCount; ++i) {
- byte[] payload = generatePayload(i);
- origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ PartHandle partHandle = buildAndPutPart(file, uploadHandle, i,
+ origDigest);
+ partHandles.put(i, partHandle);
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
payloadCount * partSizeInBytes());
}
@@ -173,17 +399,33 @@ public abstract class AbstractContractMultipartUploaderTest extends
public void testMultipartUploadEmptyPart() throws Exception {
FileSystem fs = getFileSystem();
Path file = path("testMultipartUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- MessageDigest origDigest = DigestUtils.getMd5Digest();
- byte[] payload = new byte[0];
- origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(0, partHandle));
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+ try (MultipartUploader uploader =
+ MultipartUploaderFactory.get(fs, null)) {
+ UploadHandle uploadHandle = uploader.initialize(file);
+
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ MessageDigest origDigest = DigestUtils.getMd5Digest();
+ byte[] payload = new byte[0];
+ origDigest.update(payload);
+ InputStream is = new ByteArrayInputStream(payload);
+ PartHandle partHandle = uploader.putPart(file, is, 1, uploadHandle,
+ payload.length);
+ partHandles.put(1, partHandle);
+ completeUpload(file, uploadHandle, partHandles, origDigest, 0);
+ }
+ }
+
+ /**
+ * Assert that a multipart upload is successful.
+ * @throws Exception failure
+ */
+ @Test
+ public void testUploadEmptyBlock() throws Exception {
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ partHandles.put(1, putPart(file, uploadHandle, 1, new byte[0]));
+ completeUpload(file, uploadHandle, partHandles, null, 0);
}
/**
@@ -192,11 +434,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testMultipartUploadReverseOrder() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadReverseOrder");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
final int payloadCount = getTestPayloadCount();
for (int i = 1; i <= payloadCount; ++i) {
@@ -204,13 +444,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
origDigest.update(payload);
}
for (int i = payloadCount; i > 0; --i) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
payloadCount * partSizeInBytes());
}
@@ -222,25 +458,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
public void testMultipartUploadReverseOrderNonContiguousPartNumbers()
throws Exception {
describe("Upload in reverse order and the part numbers are not contiguous");
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadReverseOrderNonContiguousPartNumbers");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
MessageDigest origDigest = DigestUtils.getMd5Digest();
int payloadCount = 2 * getTestPayloadCount();
for (int i = 2; i <= payloadCount; i += 2) {
byte[] payload = generatePayload(i);
origDigest.update(payload);
}
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
for (int i = payloadCount; i > 0; i -= 2) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
getTestPayloadCount() * partSizeInBytes());
}
@@ -251,19 +481,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
@Test
public void testMultipartUploadAbort() throws Exception {
describe("Upload and then abort it before completing");
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadAbort");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- for (int i = 20; i >= 10; --i) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ int end = 10;
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ for (int i = 12; i > 10; i--) {
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- mpu.abort(file, uploadHandle);
+ abortUpload(file, uploadHandle);
String contents = "ThisIsPart49\n";
int len = contents.getBytes(Charsets.UTF_8).length;
@@ -275,6 +500,15 @@ public abstract class AbstractContractMultipartUploaderTest extends
() -> mpu.complete(file, partHandles, uploadHandle));
assertPathDoesNotExist("Uploaded file should not exist", file);
+
+ // A second abort should be an FileNotFoundException if the UploadHandle is
+ // consumed by finalization operations (complete, abort).
+ if (finalizeConsumesUploadIdImmediately()) {
+ intercept(FileNotFoundException.class,
+ () -> abortUpload(file, uploadHandle));
+ } else {
+ abortUpload(file, uploadHandle);
+ }
}
/**
@@ -282,13 +516,23 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testAbortUnknownUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testAbortUnknownUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ Path file = methodPath();
ByteBuffer byteBuffer = ByteBuffer.wrap(
"invalid-handle".getBytes(Charsets.UTF_8));
UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
- intercept(FileNotFoundException.class, () -> mpu.abort(file, uploadHandle));
+ intercept(FileNotFoundException.class,
+ () -> abortUpload(file, uploadHandle));
+ }
+
+ /**
+ * Trying to abort with a handle of size 0 must fail.
+ */
+ @Test
+ public void testAbortEmptyUpload() throws Exception {
+ describe("initialize upload and abort before uploading data");
+ Path file = methodPath();
+ abortUpload(file, initializeUpload(file));
+ assertPathDoesNotExist("Uploaded file should not exist", file);
}
/**
@@ -296,13 +540,10 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testAbortEmptyUploadHandle() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testAbortEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[0]);
UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
intercept(IllegalArgumentException.class,
- () -> mpu.abort(file, uploadHandle));
+ () -> abortUpload(methodPath(), uploadHandle));
}
/**
@@ -311,26 +552,20 @@ public abstract class AbstractContractMultipartUploaderTest extends
@Test
public void testCompleteEmptyUpload() throws Exception {
describe("Expect an empty MPU to fail, but still be abortable");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle handle = mpu.initialize(dest);
- intercept(IOException.class,
- () -> mpu.complete(dest, new ArrayList<>(), handle));
- mpu.abort(dest, handle);
+ Path dest = methodPath();
+ UploadHandle handle = initializeUpload(dest);
+ intercept(IllegalArgumentException.class,
+ () -> mpu.complete(dest, new HashMap<>(), handle));
+ abortUpload(dest, handle);
}
/**
* When we pass empty uploadID, putPart throws IllegalArgumentException.
- * @throws Exception
*/
@Test
public void testPutPartEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when putPart uploadID is empty");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- mpu.initialize(dest);
+ Path dest = methodPath();
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
byte[] payload = generatePayload(1);
@@ -341,25 +576,123 @@ public abstract class AbstractContractMultipartUploaderTest extends
/**
* When we pass empty uploadID, complete throws IllegalArgumentException.
- * @throws Exception
*/
@Test
public void testCompleteEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when complete uploadID is empty");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle realHandle = mpu.initialize(dest);
+ Path dest = methodPath();
+ UploadHandle realHandle = initializeUpload(dest);
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- byte[] payload = generatePayload(1);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(dest, is, 1, realHandle,
- payload.length);
- partHandles.add(Pair.of(1, partHandle));
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ PartHandle partHandle = putPart(dest, realHandle, 1,
+ generatePayload(1, SMALL_FILE));
+ partHandles.put(1, partHandle);
intercept(IllegalArgumentException.class,
() -> mpu.complete(dest, partHandles, emptyHandle));
+
+ // and, while things are setup, attempt to complete with
+ // a part index of 0
+ partHandles.clear();
+ partHandles.put(0, partHandle);
+ intercept(IllegalArgumentException.class,
+ () -> mpu.complete(dest, partHandles, realHandle));
+ }
+
+ /**
+ * Assert that upon completion, a directory in the way of the file will
+ * result in a failure. This test only applies to backing stores with a
+ * concept of directories.
+ * @throws Exception failure
+ */
+ @Test
+ public void testDirectoryInTheWay() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ int size = SMALL_FILE;
+ PartHandle partHandle = putPart(file, uploadHandle, 1,
+ generatePayload(1, size));
+ partHandles.put(1, partHandle);
+
+ fs.mkdirs(file);
+ intercept(IOException.class,
+ () -> completeUpload(file, uploadHandle, partHandles, null,
+ size));
+ // abort should still work
+ abortUpload(file, uploadHandle);
+ }
+
+ @Test
+ public void testConcurrentUploads() throws Throwable {
+
+ // if the FS doesn't support concurrent uploads, this test is
+ // required to fail during the second initialization.
+ final boolean concurrent = supportsConcurrentUploadsToSamePath();
+
+ describe("testing concurrent uploads, MPU support for this is "
+ + concurrent);
+ final FileSystem fs = getFileSystem();
+ final Path file = methodPath();
+ final int size1 = SMALL_FILE;
+ final int partId1 = 1;
+ final byte[] payload1 = generatePayload(partId1, size1);
+ final MessageDigest digest1 = DigestUtils.getMd5Digest();
+ digest1.update(payload1);
+ final UploadHandle upload1 = initializeUpload(file);
+ final Map<Integer, PartHandle> partHandles1 = new HashMap<>();
+
+ // initiate part 2
+ // by using a different size, it's straightforward to see which
+ // version is visible, before reading/digesting the contents
+ final int size2 = size1 * 2;
+ final int partId2 = 2;
+ final byte[] payload2 = generatePayload(partId1, size2);
+ final MessageDigest digest2 = DigestUtils.getMd5Digest();
+ digest2.update(payload2);
+
+ final UploadHandle upload2;
+ try {
+ upload2 = initializeUpload(file);
+ Assume.assumeTrue(
+ "The Filesystem is unexpectedly supporting concurrent uploads",
+ concurrent);
+ } catch (IOException e) {
+ if (!concurrent) {
+ // this is expected, so end the test
+ LOG.debug("Expected exception raised on concurrent uploads {}", e);
+ return;
+ } else {
+ throw e;
+ }
+ }
+ final Map<Integer, PartHandle> partHandles2 = new HashMap<>();
+
+
+ assertNotEquals("Upload handles match", upload1, upload2);
+
+ // put part 1
+ partHandles1.put(partId1, putPart(file, upload1, partId1, payload1));
+
+ // put part2
+ partHandles2.put(partId2, putPart(file, upload2, partId2, payload2));
+
+ // complete part u1. expect its size and digest to
+ // be as expected.
+ completeUpload(file, upload1, partHandles1, digest1, size1);
+
+ // now upload part 2.
+ complete(file, upload2, partHandles2);
+ // and await the visible length to match
+ eventually(timeToBecomeConsistentMillis(), 500,
+ () -> {
+ FileStatus status = fs.getFileStatus(file);
+ assertEquals("File length in " + status,
+ size2, status.getLen());
+ });
+
+ verifyContents(file, digest2, size2);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
index a50d2e4..6e27964d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
@@ -40,4 +40,14 @@ public class TestLocalFSContractMultipartUploader
protected int partSizeInBytes() {
return 1024;
}
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return true;
+ }
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
index f3a5265..54f4ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
@@ -32,6 +34,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
public class TestHDFSContractMultipartUploader extends
AbstractContractMultipartUploaderTest {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
@BeforeClass
public static void createCluster() throws IOException {
HDFSContract.createCluster();
@@ -55,4 +60,14 @@ public class TestHDFSContractMultipartUploader extends
protected int partSizeInBytes() {
return 1024;
}
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return true;
+ }
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
index cab4e2a..cf58751 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
@@ -68,10 +70,8 @@ public class S3AMultipartUploader extends MultipartUploader {
public static final String HEADER = "S3A-part01";
public S3AMultipartUploader(FileSystem fs, Configuration conf) {
- if (!(fs instanceof S3AFileSystem)) {
- throw new IllegalArgumentException(
- "S3A MultipartUploads must use S3AFileSystem");
- }
+ Preconditions.checkArgument(fs instanceof S3AFileSystem,
+ "Wrong filesystem: expected S3A but got %s", fs);
s3a = (S3AFileSystem) fs;
}
@@ -88,6 +88,8 @@ public class S3AMultipartUploader extends MultipartUploader {
public PartHandle putPart(Path filePath, InputStream inputStream,
int partNumber, UploadHandle uploadId, long lengthInBytes)
throws IOException {
+ checkPutArguments(filePath, inputStream, partNumber, uploadId,
+ lengthInBytes);
byte[] uploadIdBytes = uploadId.toByteArray();
checkUploadId(uploadIdBytes);
String key = s3a.pathToKey(filePath);
@@ -105,14 +107,16 @@ public class S3AMultipartUploader extends MultipartUploader {
@Override
public PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId)
+ Map<Integer, PartHandle> handleMap,
+ UploadHandle uploadId)
throws IOException {
byte[] uploadIdBytes = uploadId.toByteArray();
checkUploadId(uploadIdBytes);
- if (handles.isEmpty()) {
- throw new IOException("Empty upload");
- }
+ checkPartHandles(handleMap);
+ List<Map.Entry<Integer, PartHandle>> handles =
+ new ArrayList<>(handleMap.entrySet());
+ handles.sort(Comparator.comparingInt(Map.Entry::getKey));
final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper();
String key = s3a.pathToKey(filePath);
@@ -121,11 +125,11 @@ public class S3AMultipartUploader extends MultipartUploader {
ArrayList<PartETag> eTags = new ArrayList<>();
eTags.ensureCapacity(handles.size());
long totalLength = 0;
- for (Pair<Integer, PartHandle> handle : handles) {
- byte[] payload = handle.getRight().toByteArray();
+ for (Map.Entry<Integer, PartHandle> handle : handles) {
+ byte[] payload = handle.getValue().toByteArray();
Pair<Long, String> result = parsePartHandlePayload(payload);
totalLength += result.getLeft();
- eTags.add(new PartETag(handle.getLeft(), result.getRight()));
+ eTags.add(new PartETag(handle.getKey(), result.getRight()));
}
AtomicInteger errorCount = new AtomicInteger(0);
CompleteMultipartUploadResult result = writeHelper.completeMPUwithRetries(
@@ -172,7 +176,7 @@ public class S3AMultipartUploader extends MultipartUploader {
throws IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(eTag),
"Empty etag");
- Preconditions.checkArgument(len > 0,
+ Preconditions.checkArgument(len >= 0,
"Invalid length");
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -190,6 +194,7 @@ public class S3AMultipartUploader extends MultipartUploader {
* @return the length and etag
* @throws IOException error reading the payload
*/
+ @VisibleForTesting
static Pair<Long, String> parsePartHandlePayload(byte[] data)
throws IOException {
@@ -201,7 +206,7 @@ public class S3AMultipartUploader extends MultipartUploader {
}
final long len = input.readLong();
final String etag = input.readUTF();
- if (len <= 0) {
+ if (len < 0) {
throw new IOException("Negative length");
}
return Pair.of(len, etag);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
index d28f39b..059312a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.fs.contract.s3a;
-import java.io.IOException;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
@@ -35,6 +34,9 @@ import static org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.DEFAULT_H
/**
* Test MultipartUploader with S3A.
+ * Although not an S3A Scale test subclass, it uses the -Dscale option
+ * to enable it, and partition size option to control the size of
+ * parts uploaded.
*/
public class ITestS3AContractMultipartUploader extends
AbstractContractMultipartUploaderTest {
@@ -79,6 +81,35 @@ public class ITestS3AContractMultipartUploader extends
return new S3AContract(conf);
}
+ /**
+ * Bigger test: use the scale timeout.
+ * @return the timeout for scale tests.
+ */
+ @Override
+ protected int getTestTimeoutMillis() {
+ return SCALE_TEST_TIMEOUT_MILLIS;
+ }
+
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
+
+ /**
+ * Provide a pessimistic time to become consistent.
+ * @return a time in milliseconds
+ */
+ @Override
+ protected int timeToBecomeConsistentMillis() {
+ return 30 * 1000;
+ }
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return false;
+ }
+
@Override
public void setup() throws Exception {
super.setup();
@@ -103,14 +134,29 @@ public class ITestS3AContractMultipartUploader extends
public void teardown() throws Exception {
Path teardown = path("teardown").getParent();
S3AFileSystem fs = getFileSystem();
- WriteOperationHelper helper = fs.getWriteOperationHelper();
- try {
- LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
- int count = helper.abortMultipartUploadsUnderPath(fs.pathToKey(teardown));
- LOG.info("Found {} incomplete uploads", count);
- } catch (IOException e) {
- LOG.warn("IOE in teardown", e);
+ if (fs != null) {
+ WriteOperationHelper helper = fs.getWriteOperationHelper();
+ try {
+ LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
+ int count = helper.abortMultipartUploadsUnderPath(
+ fs.pathToKey(teardown));
+ LOG.info("Found {} incomplete uploads", count);
+ } catch (Exception e) {
+ LOG.warn("Exeception in teardown", e);
+ }
}
super.teardown();
}
+
+ /**
+ * S3 has no concept of directories, so this test does not apply.
+ */
+ public void testDirectoryInTheWay() throws Exception {
+ // no-op
+ }
+
+ @Override
+ public void testMultipartUploadReverseOrder() throws Exception {
+ ContractTestUtils.skip("skipped for speed");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
index 35d0460..4825d26 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
@@ -60,7 +60,7 @@ public class TestS3AMultipartUploaderSupport extends HadoopTestBase {
@Test
public void testNoLen() throws Throwable {
intercept(IllegalArgumentException.class,
- () -> buildPartHandlePayload("tag", 0));
+ () -> buildPartHandlePayload("tag", -1));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDDS-836. Add TokenIdentifier
Ozone for delegation token and block token. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-836. Add TokenIdentifier Ozone for delegation token and block token. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c0baf0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c0baf0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c0baf0e
Branch: refs/heads/HDDS-4
Commit: 9c0baf0ea30a0dc38778c6030eca1468ed7e40ae
Parents: 4770e9d
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 14 14:26:33 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../hdds/security/x509/keys/SecurityUtil.java | 59 ++++
hadoop-hdds/common/src/main/proto/hdds.proto | 24 ++
.../security/OzoneBlockTokenIdentifier.java | 178 +++++++++++
.../ozone/security/OzoneBlockTokenSelector.java | 55 ++++
.../security/OzoneDelegationTokenSelector.java | 52 ++++
.../hadoop/ozone/security/OzoneSecretKey.java | 195 ++++++++++++
.../ozone/security/OzoneTokenIdentifier.java | 217 ++++++++++++++
.../hadoop/ozone/security/package-info.java | 21 ++
.../src/main/proto/OzoneManagerProtocol.proto | 20 ++
.../security/TestOzoneBlockTokenIdentifier.java | 255 ++++++++++++++++
.../security/TestOzoneTokenIdentifier.java | 300 +++++++++++++++++++
.../hadoop/ozone/security/package-info.java | 21 ++
12 files changed, 1397 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
index 2ca8825..6147d3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -18,6 +18,15 @@
*/
package org.apache.hadoop.hdds.security.x509.keys;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
@@ -76,4 +85,54 @@ public final class SecurityUtil {
}
throw new CertificateException("No PKCS#9 extension found in CSR");
}
+
+ /*
+ * Returns private key created from encoded key.
+ * @return private key if successful else returns null.
+ */
+ public static PrivateKey getPrivateKey(byte[] encodedKey,
+ SecurityConfig secureConfig) {
+ PrivateKey pvtKey = null;
+ if (encodedKey == null || encodedKey.length == 0) {
+ return null;
+ }
+
+ try {
+ KeyFactory kf = null;
+
+ kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+ secureConfig.getProvider());
+ pvtKey = kf.generatePrivate(new PKCS8EncodedKeySpec(encodedKey));
+
+ } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+ NoSuchProviderException e) {
+ return null;
+ }
+ return pvtKey;
+ }
+
+ /*
+ * Returns public key created from encoded key.
+ * @return public key if successful else returns null.
+ */
+ public static PublicKey getPublicKey(byte[] encodedKey,
+ SecurityConfig secureConfig) {
+ PublicKey key = null;
+ if (encodedKey == null || encodedKey.length == 0) {
+ return null;
+ }
+
+ try {
+ KeyFactory kf = null;
+ kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+ secureConfig.getProvider());
+ key = kf.generatePublic(new X509EncodedKeySpec(encodedKey));
+
+ } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+ NoSuchProviderException e) {
+ return null;
+ }
+ return key;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index cf3d6d4..8598fbf 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -189,6 +189,30 @@ message ContainerBlockID {
required int64 localID = 2;
}
+
+/**
+ * Information for the Hdds block token.
+ * When adding further fields, make sure they are optional as they would
+ * otherwise not be backwards compatible.
+ */
+message BlockTokenSecretProto {
+ /**
+ * File access permissions mode.
+ */
+ enum AccessModeProto {
+ READ = 1;
+ WRITE = 2;
+ COPY = 3;
+ DELETE = 4;
+ }
+ required string ownerId = 1;
+ required string blockId = 2;
+ required uint64 expiryDate = 3;
+ required string omCertSerialId = 4;
+ repeated AccessModeProto modes = 5;
+
+}
+
message BlockID {
required ContainerBlockID containerBlockID = 1;
optional uint64 blockCommitSequenceId = 2 [default = 0];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..92b6d16
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+ static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+ private long expiryDate;
+ private String ownerId;
+ private String blockId;
+ private final EnumSet<AccessModeProto> modes;
+ private final String omCertSerialId;
+
+ public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+ EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
+ this.ownerId = ownerId;
+ this.blockId = blockId;
+ this.expiryDate = expiryDate;
+ this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+ this.omCertSerialId = omCertSerialId;
+ }
+
+ @Override
+ public UserGroupInformation getUser() {
+ if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+ return UserGroupInformation.createRemoteUser(blockId);
+ }
+ return UserGroupInformation.createRemoteUser(ownerId);
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public String getOwnerId() {
+ return ownerId;
+ }
+
+ public String getBlockId() {
+ return blockId;
+ }
+
+ public EnumSet<AccessModeProto> getAccessModes() {
+ return modes;
+ }
+
+ public String getOmCertSerialId(){
+ return omCertSerialId;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String toString() {
+ return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ + ", ownerId=" + this.getOwnerId()
+ + ", omCertSerialId=" + this.getOmCertSerialId()
+ + ", blockId=" + this.getBlockId() + ", access modes="
+ + this.getAccessModes() + ")";
+ }
+
+ static boolean isEqual(Object a, Object b) {
+ return a == null ? b == null : a.equals(b);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneBlockTokenIdentifier) {
+ OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+ return new EqualsBuilder()
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.ownerId, that.ownerId)
+ .append(this.blockId, that.blockId)
+ .append(this.modes, that.modes)
+ .append(this.omCertSerialId, that.omCertSerialId)
+ .build();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(133, 567)
+ .append(this.expiryDate)
+ .append(this.blockId)
+ .append(this.ownerId)
+ .append(this.modes)
+ .append(this.omCertSerialId)
+ .build();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final DataInputStream dis = (DataInputStream) in;
+ if (!dis.markSupported()) {
+ throw new IOException("Could not peek first byte.");
+ }
+ readFieldsProtobuf(dis);
+ }
+
+ @VisibleForTesting
+ public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+ throws IOException {
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+ tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+ tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ writeProtobuf(out);
+ }
+
+ @VisibleForTesting
+ void writeProtobuf(DataOutput out) throws IOException {
+ Builder builder = BlockTokenSecretProto.newBuilder()
+ .setBlockId(this.getBlockId())
+ .setOwnerId(this.getOwnerId())
+ .setOmCertSerialId(this.getOmCertSerialId())
+ .setExpiryDate(this.getExpiryDate());
+ // Add access mode allowed
+ for (AccessModeProto mode : this.getAccessModes()) {
+ builder.addModes(AccessModeProto.valueOf(mode.name()));
+ }
+ out.write(builder.build().toByteArray());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..086e19b
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+ TokenSelector<OzoneBlockTokenIdentifier> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneBlockTokenSelector.class);
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+ LOG.trace("Getting token for service:{}", service);
+ return (Token<OzoneBlockTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
new file mode 100644
index 0000000..0d63c4e
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.util.Collection;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A delegation token selector that is specialized for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneDelegationTokenSelector
+ extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+
+ public OzoneDelegationTokenSelector() {
+ super(OzoneTokenIdentifier.KIND_NAME);
+ }
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneDelegationTokenSelector.class);
+
+ @Override
+ public Token selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ LOG.trace("Getting token for service {}", service);
+ return super.selectToken(service, tokens);
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
new file mode 100644
index 0000000..9b2f912
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SecretKeyProto;
+
+/**
+ * Wrapper class for Ozone/Hdds secret keys. Used in delegation tokens and block
+ * tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretKey implements Writable {
+
+ private int keyId;
+ private long expiryDate;
+ private PrivateKey privateKey;
+ private PublicKey publicKey;
+ private int maxKeyLen;
+ private SecurityConfig securityConfig;
+
+ public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair,
+ int maxKeyLen) {
+ Preconditions.checkNotNull(keyId);
+ this.keyId = keyId;
+ this.expiryDate = expiryDate;
+ byte[] encodedKey = keyPair.getPrivate().getEncoded();
+ this.maxKeyLen = maxKeyLen;
+ if (encodedKey.length > maxKeyLen) {
+ throw new RuntimeException("can't create " + encodedKey.length +
+ " byte long DelegationKey.");
+ }
+ this.privateKey = keyPair.getPrivate();
+ this.publicKey = keyPair.getPublic();
+ }
+
+ /*
+ * Create new instance using default signature algorithm and provider.
+ * */
+ public OzoneSecretKey(int keyId, long expiryDate, byte[] pvtKey,
+ byte[] publicKey, int maxKeyLen) {
+ Preconditions.checkNotNull(pvtKey);
+ Preconditions.checkNotNull(publicKey);
+
+ this.securityConfig = new SecurityConfig(new OzoneConfiguration());
+ this.keyId = keyId;
+ this.expiryDate = expiryDate;
+ this.maxKeyLen = maxKeyLen;
+ if (pvtKey.length > maxKeyLen) {
+ throw new RuntimeException("can't create " + pvtKey.length +
+ " byte long DelegationKey. Max allowed length is " + maxKeyLen);
+ }
+ this.privateKey = SecurityUtil.getPrivateKey(pvtKey, securityConfig);
+ this.publicKey = SecurityUtil.getPublicKey(publicKey, securityConfig);
+ }
+
+ public int getKeyId() {
+ return keyId;
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public PrivateKey getPrivateKey() {
+ return privateKey;
+ }
+
+ public PublicKey getPublicKey() {
+ return publicKey;
+ }
+
+ public int getMaxKeyLen() {
+ return maxKeyLen;
+ }
+
+ public byte[] getEncodedPrivateKey() {
+ return privateKey.getEncoded();
+ }
+
+ public byte[] getEncodedPubliceKey() {
+ return publicKey.getEncoded();
+ }
+
+ public void setExpiryDate(long expiryDate) {
+ this.expiryDate = expiryDate;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ SecretKeyProto token = SecretKeyProto.newBuilder()
+ .setKeyId(getKeyId())
+ .setExpiryDate(getExpiryDate())
+ .setPrivateKeyBytes(ByteString.copyFrom(getEncodedPrivateKey()))
+ .setPublicKeyBytes(ByteString.copyFrom(getEncodedPubliceKey()))
+ .setMaxKeyLen(getMaxKeyLen())
+ .build();
+ out.write(token.toByteArray());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ SecretKeyProto secretKey = SecretKeyProto.parseFrom((DataInputStream) in);
+ expiryDate = secretKey.getExpiryDate();
+ keyId = secretKey.getKeyId();
+ privateKey = SecurityUtil.getPrivateKey(secretKey.getPrivateKeyBytes()
+ .toByteArray(), securityConfig);
+ publicKey = SecurityUtil.getPublicKey(secretKey.getPublicKeyBytes()
+ .toByteArray(), securityConfig);
+ maxKeyLen = secretKey.getMaxKeyLen();
+ }
+
+ @Override
+ public int hashCode() {
+ HashCodeBuilder hashCodeBuilder = new HashCodeBuilder(537, 963);
+ hashCodeBuilder.append(getExpiryDate())
+ .append(getKeyId())
+ .append(getEncodedPrivateKey())
+ .append(getEncodedPubliceKey());
+
+ return hashCodeBuilder.build();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneSecretKey) {
+ OzoneSecretKey that = (OzoneSecretKey) obj;
+ return new EqualsBuilder()
+ .append(this.keyId, that.keyId)
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.privateKey, that.privateKey)
+ .append(this.publicKey, that.publicKey)
+ .build();
+ }
+ return false;
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+ */
+ static OzoneSecretKey readProtoBuf(DataInput in) throws IOException {
+ Preconditions.checkNotNull(in);
+ SecretKeyProto key = SecretKeyProto.parseFrom((DataInputStream) in);
+ return new OzoneSecretKey(key.getKeyId(), key.getExpiryDate(),
+ key.getPrivateKeyBytes().toByteArray(),
+ key.getPublicKeyBytes().toByteArray(), key.getMaxKeyLen());
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+ */
+ static OzoneSecretKey readProtoBuf(byte[] identifier) throws IOException {
+ Preconditions.checkNotNull(identifier);
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+ identifier));
+ return readProtoBuf(in);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
new file mode 100644
index 0000000..d053465
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * The token identifier for Ozone Master.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneTokenIdentifier extends
+ AbstractDelegationTokenIdentifier {
+
+ public final static Text KIND_NAME = new Text("OzoneToken");
+
+ /**
+ * Create an empty delegation token identifier.
+ */
+ public OzoneTokenIdentifier() {
+ super();
+ }
+
+ /**
+ * Create a new ozone master delegation token identifier.
+ *
+ * @param owner the effective username of the token owner
+ * @param renewer the username of the renewer
+ * @param realUser the real username of the token owner
+ */
+ public OzoneTokenIdentifier(Text owner, Text renewer, Text realUser) {
+ super(owner, renewer, realUser);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ /**
+ * Default TrivialRenewer.
+ */
+ @InterfaceAudience.Private
+ public static class Renewer extends Token.TrivialRenewer {
+
+ @Override
+ protected Text getKind() {
+ return KIND_NAME;
+ }
+ }
+
+ /**
+ * Overrides default implementation to write using Protobuf.
+ *
+ * @param out output stream
+ * @throws IOException
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ OMTokenProto token = OMTokenProto.newBuilder()
+ .setOwner(getOwner().toString())
+ .setRealUser(getRealUser().toString())
+ .setRenewer(getRenewer().toString())
+ .setIssueDate(getIssueDate())
+ .setMaxDate(getMaxDate())
+ .setSequenceNumber(getSequenceNumber())
+ .setMasterKeyId(getMasterKeyId()).build();
+ out.write(token.toByteArray());
+ }
+
+ /**
+ * Overrides default implementation to read using Protobuf.
+ *
+ * @param in input stream
+ * @throws IOException
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ setOwner(new Text(token.getOwner()));
+ setRealUser(new Text(token.getRealUser()));
+ setRenewer(new Text(token.getRenewer()));
+ setIssueDate(token.getIssueDate());
+ setMaxDate(token.getMaxDate());
+ setSequenceNumber(token.getSequenceNumber());
+ setMasterKeyId(token.getMasterKeyId());
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link
+ * OzoneTokenIdentifier}.
+ */
+ public static OzoneTokenIdentifier readProtoBuf(DataInput in)
+ throws IOException {
+ OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ OzoneTokenIdentifier identifier = new OzoneTokenIdentifier();
+ identifier.setRenewer(new Text(token.getRenewer()));
+ identifier.setOwner(new Text(token.getOwner()));
+ identifier.setRealUser(new Text(token.getRealUser()));
+ identifier.setMaxDate(token.getMaxDate());
+ identifier.setIssueDate(token.getIssueDate());
+ identifier.setSequenceNumber(token.getSequenceNumber());
+ identifier.setMasterKeyId(token.getMasterKeyId());
+ return identifier;
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link
+ * OzoneTokenIdentifier}.
+ */
+ public static OzoneTokenIdentifier readProtoBuf(byte[] identifier)
+ throws IOException {
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+ identifier));
+ return readProtoBuf(in);
+ }
+
+ /**
+ * Creates new instance.
+ */
+ public static OzoneTokenIdentifier newInstance() {
+ return new OzoneTokenIdentifier();
+ }
+
+ /**
+ * Creates new instance.
+ */
+ public static OzoneTokenIdentifier newInstance(Text owner, Text renewer,
+ Text realUser) {
+ return new OzoneTokenIdentifier(owner, renewer, realUser);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof OzoneTokenIdentifier)) {
+ return false;
+ }
+ return super.equals(obj);
+ }
+
+ /**
+ * Class to encapsulate a token's renew date and password.
+ */
+ @InterfaceStability.Evolving
+ public static class TokenInfo {
+
+ private long renewDate;
+ private byte[] password;
+ private String trackingId;
+
+ public TokenInfo(long renewDate, byte[] password) {
+ this(renewDate, password, null);
+ }
+
+ public TokenInfo(long renewDate, byte[] password,
+ String trackingId) {
+ this.renewDate = renewDate;
+ this.password = Arrays.copyOf(password, password.length);
+ this.trackingId = trackingId;
+ }
+
+ /**
+ * returns renew date.
+ */
+ public long getRenewDate() {
+ return renewDate;
+ }
+
+ /**
+ * returns password.
+ */
+ byte[] getPassword() {
+ return password;
+ }
+
+ /**
+ * returns tracking id.
+ */
+ public String getTrackingId() {
+ return trackingId;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..457f891
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security related classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 6320cf1..5caebf5 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -311,6 +311,26 @@ message DeleteBucketResponse {
required Status status = 1;
}
+message OMTokenProto {
+ optional uint32 version = 1;
+ optional string owner = 2;
+ optional string renewer = 3;
+ optional string realUser = 4;
+ optional uint64 issueDate = 5;
+ optional uint64 maxDate = 6;
+ optional uint32 sequenceNumber = 7;
+ optional uint32 masterKeyId = 8;
+ optional uint64 expiryDate = 9;
+}
+
+message SecretKeyProto {
+ required uint32 keyId = 1;
+ required uint64 expiryDate = 2;
+ required bytes privateKeyBytes = 3;
+ required bytes publicKeyBytes = 4;
+ required uint32 maxKeyLen = 5;
+}
+
message ListKeysRequest {
required string volumeName = 1;
required string bucketName = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..5909916
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneBlockTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static long expiryTime;
+ private static KeyPair keyPair;
+ private static X509Certificate cert;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneBlockTokenIdentifier("", "",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValid = rsaSignature.verify(signature);
+ return isValid;
+ }
+
+ private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneBlockTokenIdentifier generateTestToken() {
+ return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+ RandomStringUtils.randomAlphabetic(5),
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate certificate;
+ certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ kp, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration / testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+ certificate);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration / testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration / testTokenCount);
+ }
+
+ // TODO: verify certificate with a trust store
+ public boolean verifyCert(Certificate certificate) {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
new file mode 100644
index 0000000..dfdf14b
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.TestSSLFactory;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for {@link OzoneTokenIdentifier}.
+ */
+public class TestOzoneTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static File base;
+ private static String sslConfsDir;
+ private static final String EXCLUDE_CIPHERS =
+ "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, \n"
+ + "SSL_RSA_WITH_DES_CBC_SHA,"
+ + "SSL_DHE_RSA_WITH_DES_CBC_SHA, "
+ + "SSL_RSA_EXPORT_WITH_RC4_40_MD5,\t \n"
+ + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ + "SSL_RSA_WITH_RC4_128_MD5";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ }
+
+ private Configuration createConfiguration(boolean clientCert,
+ boolean trustStore)
+ throws Exception {
+ Configuration conf = new Configuration();
+ KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
+ clientCert, trustStore, EXCLUDE_CIPHERS);
+ sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
+ return conf;
+ }
+
+ @AfterClass
+ static public void cleanUp() throws Exception {
+ FileUtil.fullyDelete(base);
+ KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+ // Create Ozone Master key pair
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneTokenIdentifier tokenId = new OzoneTokenIdentifier();
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneTokenIdentifier(new Text("oozie"),
+ new Text("rm"), new Text("client"));
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ public byte[] signTokenAsymmetric(OzoneTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValide = rsaSignature.verify(signature);
+ return isValide;
+ }
+
+ private byte[] signTokenSymmetric(OzoneTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneTokenIdentifier generateTestToken() {
+ return new OzoneTokenIdentifier(
+ new Text(RandomStringUtils.randomAlphabetic(6)),
+ new Text(RandomStringUtils.randomAlphabetic(5)),
+ new Text(RandomStringUtils.randomAlphabetic(4)));
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate cert;
+ cert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ keyPair, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), keyPair.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration/testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), cert);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration/testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration/testTokenCount);
+ }
+
+ /*
+ * Test serialization/deserialization of OzoneTokenIdentifier.
+ */
+ @Test
+ public void testReadWriteInProtobuf() throws IOException {
+ OzoneTokenIdentifier id = getIdentifierInst();
+ File idFile = new File(BASEDIR + "/tokenFile");
+
+ FileOutputStream fop = new FileOutputStream(idFile);
+ DataOutputStream dataOutputStream = new DataOutputStream(fop);
+ id.write(dataOutputStream);
+ fop.close();
+
+ FileInputStream fis = new FileInputStream(idFile);
+ DataInputStream dis = new DataInputStream(fis);
+ OzoneTokenIdentifier id2 = new OzoneTokenIdentifier();
+
+ id2.readFields(dis);
+ Assert.assertEquals(id, id2);
+ }
+
+
+ public OzoneTokenIdentifier getIdentifierInst() {
+ OzoneTokenIdentifier id = new OzoneTokenIdentifier();
+ id.setOwner(new Text("User1"));
+ id.setRenewer(new Text("yarn"));
+ id.setIssueDate(Time.now());
+ id.setMaxDate(Time.now() + 5000);
+ id.setSequenceNumber(1);
+ return id;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c0baf0e/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..a36f325
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security tests.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth.
Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcd705a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcd705a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcd705a5
Branch: refs/heads/HDDS-4
Commit: fcd705a59420dcb5d6a05e2d49536a77ae4f1d5b
Parents: 2119be4
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:56:49 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd705a5/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index d3e352b..4d79e8c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -198,6 +198,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -255,6 +256,9 @@
</property>
<property>
<name>hdds.container.report.interval</name>
+=======
+ <name>ozone.container.report.interval</name>
+>>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
<value>60000ms</value>
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>Time interval of the datanode to send container report. Each
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-101. SCM CA: generate CSR for
SCM CA clients. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-101. SCM CA: generate CSR for SCM CA clients.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1842ced1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1842ced1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1842ced1
Branch: refs/heads/HDDS-4
Commit: 1842ced1268acdf59accb54aec664043bf07da73
Parents: 7246898
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Oct 26 17:57:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../certificates/CertificateSignRequest.java | 245 +++++++++++++++++
.../security/x509/keys/HDDSKeyGenerator.java | 2 +-
.../hdds/security/x509/keys/SecurityUtil.java | 79 ++++++
.../TestCertificateSignRequest.java | 268 +++++++++++++++++++
4 files changed, 593 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
new file mode 100644
index 0000000..2e1f9df
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CA.
+ */
+public final class CertificateSignRequest {
+ private final KeyPair keyPair;
+ private final SecurityConfig config;
+ private final Extensions extensions;
+ private String subject;
+ private String clusterID;
+ private String scmID;
+
+ /**
+ * Private Ctor for CSR.
+ *
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param keyPair - KeyPair
+ * @param config - SCM Config
+ * @param extensions - CSR extensions
+ */
+ private CertificateSignRequest(String subject, String scmID, String clusterID,
+ KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.keyPair = keyPair;
+ this.config = config;
+ this.extensions = extensions;
+ }
+
+ private PKCS10CertificationRequest generateCSR() throws
+ OperatorCreationException {
+ X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+ clusterID);
+ PKCS10CertificationRequestBuilder p10Builder =
+ new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider())
+ .build(keyPair.getPrivate());
+
+ if (extensions != null) {
+ p10Builder.addAttribute(
+ PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+ }
+ return p10Builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Certificate Sign Request.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private KeyPair key;
+ private SecurityConfig config;
+ private List<GeneralName> altNames;
+ private Boolean ca = false;
+
+ public CertificateSignRequest.Builder setConfiguration(
+ Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ // Support SAN extenion with DNS and RFC822 Name
+ // other name type will be added as needed.
+ public CertificateSignRequest.Builder addDnsName(String dnsName) {
+ Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+ this.addAltName(GeneralName.dNSName, dnsName);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder addRfc822Name(String name) {
+ Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+ this.addAltName(GeneralName.rfc822Name, name);
+ return this;
+ }
+
+ // IP address is subject to change which is optional for now.
+ public CertificateSignRequest.Builder addIpAddress(String ip) {
+ Preconditions.checkNotNull(ip, "Ip address cannot be null");
+ this.addAltName(GeneralName.iPAddress, ip);
+ return this;
+ }
+
+ private CertificateSignRequest.Builder addAltName(int tag, String name) {
+ if (altNames == null) {
+ altNames = new ArrayList<>();
+ }
+ altNames.add(new GeneralName(tag, name));
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setCA(Boolean isCA) {
+ this.ca = isCA;
+ return this;
+ }
+
+ private Extension getKeyUsageExtension() throws IOException {
+ int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+ | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+ if (ca) {
+ keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+ }
+ KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+ return new Extension(Extension.keyUsage, true,
+ new DEROctetString(keyUsage));
+ }
+
+ private Optional<Extension> getSubjectAltNameExtension() throws
+ IOException {
+ if (altNames != null) {
+ return Optional.of(new Extension(Extension.subjectAlternativeName,
+ true, new DEROctetString(new GeneralNames(
+ altNames.toArray(new GeneralName[altNames.size()])))));
+ }
+ return Optional.empty();
+ }
+
+ private Extension getBasicExtension() throws IOException {
+ // We don't set pathLenConstraint means no limit is imposed.
+ return new Extension(Extension.basicConstraints,
+ true, new DEROctetString(new BasicConstraints(ca)));
+ }
+
+ private Extensions createExtensions() throws IOException {
+ List<Extension> extensions = new ArrayList<>();
+
+ // Add basic extension
+ extensions.add(getBasicExtension());
+
+ // Add key usage extension
+ extensions.add(getKeyUsageExtension());
+
+ // Add subject alternate name extension
+ Optional<Extension> san = getSubjectAltNameExtension();
+ if (san.isPresent()) {
+ extensions.add(san.get());
+ }
+
+ return new Extensions(
+ extensions.toArray(new Extension[extensions.size()]));
+ }
+
+ public PKCS10CertificationRequest build() throws SCMSecurityException {
+ Preconditions.checkNotNull(key, "KeyPair cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ try {
+ CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+ clusterID, key, config, createExtensions());
+ return csr.generateCSR();
+ } catch (IOException ioe) {
+ throw new CertificateException(String.format("Unable to create " +
+ "extension for certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+ } catch (OperatorCreationException ex) {
+ throw new CertificateException(String.format("Unable to create " +
+ "certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)),
+ ex.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 99873cb..459dce7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -96,7 +96,7 @@ public class HDDSKeyGenerator {
*/
public KeyPair generateKey(int size, String algorithm, String provider)
throws NoSuchProviderException, NoSuchAlgorithmException {
- LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ LOG.debug("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
size, algorithm, provider);
KeyPairGenerator generator = KeyPairGenerator
.getInstance(algorithm, provider);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
new file mode 100644
index 0000000..2ca8825
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.ASN1Set;
+import org.bouncycastle.asn1.pkcs.Attribute;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+
+/**
+ * Utility functions for Security modules for Ozone.
+ */
+public final class SecurityUtil {
+
+ // Ozone Certificate distinguished format: (CN=Subject,OU=ScmID,O=ClusterID).
+ private static final String DISTINGUISHED_NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+
+ private SecurityUtil() {
+ }
+
+ public static String getDistinguishedNameFormat() {
+ return DISTINGUISHED_NAME_FORMAT;
+ }
+
+ public static X500Name getDistinguishedName(String subject, String scmID,
+ String clusterID) {
+ return new X500Name(String.format(getDistinguishedNameFormat(), subject,
+ scmID, clusterID));
+ }
+
+ // TODO: move the PKCS10CSRValidator class
+ public static Extensions getPkcs9Extensions(PKCS10CertificationRequest csr)
+ throws CertificateException {
+ ASN1Set pkcs9ExtReq = getPkcs9ExtRequest(csr);
+ Object extReqElement = pkcs9ExtReq.getObjects().nextElement();
+ if (extReqElement instanceof Extensions) {
+ return (Extensions) extReqElement;
+ } else {
+ if (extReqElement instanceof ASN1Sequence) {
+ return Extensions.getInstance((ASN1Sequence) extReqElement);
+ } else {
+ throw new CertificateException("Unknown element type :" + extReqElement
+ .getClass().getSimpleName());
+ }
+ }
+ }
+
+ public static ASN1Set getPkcs9ExtRequest(PKCS10CertificationRequest csr)
+ throws CertificateException {
+ for (Attribute attr : csr.getAttributes()) {
+ ASN1ObjectIdentifier oid = attr.getAttrType();
+ if (oid.equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) {
+ return attr.getAttrValues();
+ }
+ }
+ throw new CertificateException("No PKCS#9 extension found in CSR");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1842ced1/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
new file mode 100644
index 0000000..a9285df
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -0,0 +1,268 @@
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.operator.ContentVerifierProvider;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCSException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+public class TestCertificateSignRequest {
+
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testGenerateCSR() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify basic constraints extension
+ Extension basicExt = extensions.getExtension(Extension
+ .basicConstraints);
+ Assert.assertEquals(true, basicExt.isCritical());
+
+ // Verify key usage extension
+ Extension keyUsageExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, keyUsageExt.isCritical());
+
+
+ // Verify San extension not set
+ Assert.assertEquals(null,
+ extensions.getExtension(Extension.subjectAlternativeName));
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRwithSan() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ // Multi-home
+ builder.addIpAddress("192.168.1.1");
+ builder.addIpAddress("192.168.2.1");
+
+ builder.addDnsName("dn1.abc.com");
+ builder.addRfc822Name("test@abc.com");
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify key usage extension
+ Extension sanExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, sanExt.isCritical());
+
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with blank/null Subject.
+ try {
+ builder.setSubject(null);
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+ // Now try with invalid IP address
+ try {
+ builder.addIpAddress("255.255.255.*");
+ builder.build();
+ Assert.fail("Invalid ip address");
+ } catch (IllegalArgumentException e) {
+ }
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ }
+
+ @Test
+ public void testCsrSerialization() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException, IOException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+ byte[] csrBytes = csr.getEncoded();
+
+ // Verify de-serialized CSR matches with the original CSR
+ PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes);
+ Assert.assertEquals(csr, dsCsr);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDDS-808. Simplify OMAction and
DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.
Posted by xy...@apache.org.
HDDS-808. Simplify OMAction and DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/184cced5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/184cced5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/184cced5
Branch: refs/heads/HDDS-4
Commit: 184cced513c5599d7b33c9124692fbcd2e6d338e
Parents: 07142f5
Author: Ajay Kumar <aj...@apache.org>
Authored: Thu Nov 29 08:35:02 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Thu Nov 29 08:35:20 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/audit/DNAction.java | 44 +++++++---------
.../apache/hadoop/ozone/audit/DummyAction.java | 36 ++++++-------
.../org/apache/hadoop/ozone/audit/OMAction.java | 54 +++++++++-----------
3 files changed, 58 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
index ce34c46..1c87f2b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -21,34 +21,28 @@ package org.apache.hadoop.ozone.audit;
*/
public enum DNAction implements AuditAction {
- CREATE_CONTAINER("CREATE_CONTAINER"),
- READ_CONTAINER("READ_CONTAINER"),
- UPDATE_CONTAINER("UPDATE_CONTAINER"),
- DELETE_CONTAINER("DELETE_CONTAINER"),
- LIST_CONTAINER("LIST_CONTAINER"),
- PUT_BLOCK("PUT_BLOCK"),
- GET_BLOCK("GET_BLOCK"),
- DELETE_BLOCK("DELETE_BLOCK"),
- LIST_BLOCK("LIST_BLOCK"),
- READ_CHUNK("READ_CHUNK"),
- DELETE_CHUNK("DELETE_CHUNK"),
- WRITE_CHUNK("WRITE_CHUNK"),
- LIST_CHUNK("LIST_CHUNK"),
- COMPACT_CHUNK("COMPACT_CHUNK"),
- PUT_SMALL_FILE("PUT_SMALL_FILE"),
- GET_SMALL_FILE("GET_SMALL_FILE"),
- CLOSE_CONTAINER("CLOSE_CONTAINER"),
- GET_COMMITTED_BLOCK_LENGTH("GET_COMMITTED_BLOCK_LENGTH");
-
- private String action;
-
- DNAction(String action) {
- this.action = action;
- }
+ CREATE_CONTAINER,
+ READ_CONTAINER,
+ UPDATE_CONTAINER,
+ DELETE_CONTAINER,
+ LIST_CONTAINER,
+ PUT_BLOCK,
+ GET_BLOCK,
+ DELETE_BLOCK,
+ LIST_BLOCK,
+ READ_CHUNK,
+ DELETE_CHUNK,
+ WRITE_CHUNK,
+ LIST_CHUNK,
+ COMPACT_CHUNK,
+ PUT_SMALL_FILE,
+ GET_SMALL_FILE,
+ CLOSE_CONTAINER,
+ GET_COMMITTED_BLOCK_LENGTH;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
index 76cd39a..d2da3e6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
@@ -22,30 +22,24 @@ package org.apache.hadoop.ozone.audit;
*/
public enum DummyAction implements AuditAction {
- CREATE_VOLUME("CREATE_VOLUME"),
- CREATE_BUCKET("CREATE_BUCKET"),
- CREATE_KEY("CREATE_KEY"),
- READ_VOLUME("READ_VOLUME"),
- READ_BUCKET("READ_BUCKET"),
- READ_KEY("READ_BUCKET"),
- UPDATE_VOLUME("UPDATE_VOLUME"),
- UPDATE_BUCKET("UPDATE_BUCKET"),
- UPDATE_KEY("UPDATE_KEY"),
- DELETE_VOLUME("DELETE_VOLUME"),
- DELETE_BUCKET("DELETE_BUCKET"),
- DELETE_KEY("DELETE_KEY"),
- SET_OWNER("SET_OWNER"),
- SET_QUOTA("SET_QUOTA");
-
- private final String action;
-
- DummyAction(String action) {
- this.action = action;
- }
+ CREATE_VOLUME,
+ CREATE_BUCKET,
+ CREATE_KEY,
+ READ_VOLUME,
+ READ_BUCKET,
+ READ_KEY,
+ UPDATE_VOLUME,
+ UPDATE_BUCKET,
+ UPDATE_KEY,
+ DELETE_VOLUME,
+ DELETE_BUCKET,
+ DELETE_KEY,
+ SET_OWNER,
+ SET_QUOTA;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index 1d4d646..8794014 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -23,40 +23,34 @@ package org.apache.hadoop.ozone.audit;
public enum OMAction implements AuditAction {
// WRITE Actions
- ALLOCATE_BLOCK("ALLOCATE_BLOCK"),
- ALLOCATE_KEY("ALLOCATE_KEY"),
- COMMIT_KEY("COMMIT_KEY"),
- CREATE_VOLUME("CREATE_VOLUME"),
- CREATE_BUCKET("CREATE_BUCKET"),
- CREATE_KEY("CREATE_KEY"),
- DELETE_VOLUME("DELETE_VOLUME"),
- DELETE_BUCKET("DELETE_BUCKET"),
- DELETE_KEY("DELETE_KEY"),
- RENAME_KEY("RENAME_KEY"),
- SET_OWNER("SET_OWNER"),
- SET_QUOTA("SET_QUOTA"),
- UPDATE_VOLUME("UPDATE_VOLUME"),
- UPDATE_BUCKET("UPDATE_BUCKET"),
- UPDATE_KEY("UPDATE_KEY"),
+ ALLOCATE_BLOCK,
+ ALLOCATE_KEY,
+ COMMIT_KEY,
+ CREATE_VOLUME,
+ CREATE_BUCKET,
+ CREATE_KEY,
+ DELETE_VOLUME,
+ DELETE_BUCKET,
+ DELETE_KEY,
+ RENAME_KEY,
+ SET_OWNER,
+ SET_QUOTA,
+ UPDATE_VOLUME,
+ UPDATE_BUCKET,
+ UPDATE_KEY,
// READ Actions
- CHECK_VOLUME_ACCESS("CHECK_VOLUME_ACCESS"),
- LIST_BUCKETS("LIST_BUCKETS"),
- LIST_VOLUMES("LIST_VOLUMES"),
- LIST_KEYS("LIST_KEYS"),
- READ_VOLUME("READ_VOLUME"),
- READ_BUCKET("READ_BUCKET"),
- READ_KEY("READ_BUCKET"),
- LIST_S3BUCKETS("LIST_S3BUCKETS");
-
- private String action;
-
- OMAction(String action) {
- this.action = action;
- }
+ CHECK_VOLUME_ACCESS,
+ LIST_BUCKETS,
+ LIST_VOLUMES,
+ LIST_KEYS,
+ READ_VOLUME,
+ READ_BUCKET,
+ READ_KEY,
+ LIST_S3BUCKETS;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: Fix merge conflicts
Posted by xy...@apache.org.
Fix merge conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b802cb5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b802cb5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b802cb5b
Branch: refs/heads/HDDS-4
Commit: b802cb5b039172111a815f9ba8c8415f2d402fb8
Parents: ed10fa6
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Jul 31 18:17:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 2 -
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 ++-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../common/src/main/resources/ozone-default.xml | 21 ++---
.../scm/server/StorageContainerManager.java | 9 ++-
.../StorageContainerManagerHttpServer.java | 4 +-
.../src/test/compose/compose-secure/.env | 2 +-
.../test/compose/compose-secure/docker-config | 55 ++++++++++---
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 --------------------
.../apache/hadoop/ozone/MiniOzoneCluster.java | 3 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 22 ++---
12 files changed, 87 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index bce1cd5..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,8 +436,6 @@ public abstract class TestConfigurationFieldsBase {
// Create XML key/value map
LOG_XML.debug("Reading XML property files\n");
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
- // Remove hadoop property set in ozone-default.xml
- xmlKeyValueMap.remove("hadoop.custom.tags");
LOG_XML.debug("\n=====\n");
// Create default configuration variable key/value map
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 376e6db..cb6672f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -313,10 +313,12 @@ public final class ScmConfigKeys {
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
- "ozone.scm.web.authentication.kerberos.principal";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.scm.web.authentication.kerberos.keytab";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+ "hdds.scm.web.authentication.kerberos.principal";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.web.authentication.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 578a983..b47113b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -334,6 +334,7 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
+
/**
* Config properties to set client side checksum properties.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0c99368..7d91dc8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -500,7 +500,6 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
@@ -510,9 +509,6 @@
</property>
<property>
<name>ozone.om.db.cache.size.mb</name>
-=======
- <name>ozone.ksm.db.cache.size.mb</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>128</value>
<tag>OM, PERFORMANCE</tag>
<description>
@@ -1023,7 +1019,7 @@
</property>
<property>
- <name>hdds.scm.container.creation.lease.timeout</name>
+ <name>ozone.scm.container.creation.lease.timeout</name>
<value>60s</value>
<tag>OZONE, SCM</tag>
<description>
@@ -1077,11 +1073,7 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>hdds.container.close.threshold</name>
-=======
- <name>hdds.scm.container.close.threshold</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>0.9f</value>
<tag>OZONE, DATANODE</tag>
<description>
@@ -1285,8 +1277,8 @@
datanode unless the datanode confirms the completion.
</description>
</property>
+
<property>
-<<<<<<< HEAD
<name>hdds.db.profile</name>
<value>DISK</value>
<tag>OZONE, OM, PERFORMANCE</tag>
@@ -1294,6 +1286,7 @@
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
</property>
+
<property>
<name>hdds.datanode.replication.work.dir</name>
<tag>DATANODE</tag>
@@ -1395,6 +1388,14 @@
file. Unit could be defined with postfix (ns,ms,s,m,h,d)
</description>
</property>
+ <property>
+ <name>ozone.security.enabled</name>
+ <value>false</value>
+ <tag> OZONE, SECURITY, FLAG</tag>
+ <description>True if security is enabled for ozone. When this property is
+ true, hadoop.security.authentication should be Kerberos.
+ </description>
+ </property>
<property>
<name>hdds.scm.kerberos.keytab.file</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 27fe326..2d5b330 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -442,11 +442,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param args command line arguments.
* @param conf HDDS configuration
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
@VisibleForTesting
- public static StorageContainerManager createSCM(String[] args,
- OzoneConfiguration conf) throws IOException, AuthenticationException {
+ public static StorageContainerManager createSCM(
+ String[] args, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -457,7 +458,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param conf HDDS configuration
* @param printBanner if true, then log a verbose startup message.
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
private static StorageContainerManager createSCM(
String[] args,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index da936ad..41dd89a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
index 3254735..199d52c 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
+OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 360b69a..cdf72ce 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -22,7 +22,7 @@ OZONE-SITE.XML_hdds.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
@@ -32,21 +32,12 @@ OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.C
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.block.access.token.enable=true
-HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
@@ -55,9 +46,51 @@ LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn om scm HTTP testuser
+KERBEROS_KEYTABS=dn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index cc25dbe..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
- /**
- * Never constructed.
- */
- private KSMConfigKeys() {
- }
-
-
- public static final String OZONE_KSM_HANDLER_COUNT_KEY =
- "ozone.ksm.handler.count.key";
- public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
- public static final String OZONE_KSM_ADDRESS_KEY =
- "ozone.ksm.address";
- public static final String OZONE_KSM_BIND_HOST_DEFAULT =
- "0.0.0.0";
- public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
- public static final String OZONE_KSM_HTTP_ENABLED_KEY =
- "ozone.ksm.http.enabled";
- public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
- "ozone.ksm.http-bind-host";
- public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
- "ozone.ksm.https-bind-host";
- public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
- "ozone.ksm.http-address";
- public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
- "ozone.ksm.https-address";
- public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
- public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
- public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
- "ozone.ksm.db.cache.size.mb";
- public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_KSM_USER_MAX_VOLUME =
- "ozone.ksm.user.max.volume";
- public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
- // KSM Default user/group permissions
- public static final String OZONE_KSM_USER_RIGHTS =
- "ozone.ksm.user.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KSM_GROUP_RIGHTS =
- "ozone.ksm.group.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
- "ozone.key.deleting.limit.per.task";
- public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.om.web.authentication.kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
- "ozone.om.web.authentication.kerberos.keytab";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3fe5b76..cbed243 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -150,7 +150,8 @@ public interface MiniOzoneCluster {
* @throws TimeoutException
* @throws InterruptedException
*/
- void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
+ void restartStorageContainerManager() throws InterruptedException,
+ TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 44cbecc..d7f74b9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -233,8 +233,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
- public void restartStorageContainerManager() throws TimeoutException,
- InterruptedException, IOException, AuthenticationException {
+ public void restartStorageContainerManager()
+ throws TimeoutException, InterruptedException, IOException,
+ AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b802cb5b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 3fb5499..14c6800 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -89,10 +89,6 @@ public final class TestSecureOzoneCluster {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
-
- clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
- omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -120,11 +116,12 @@ public final class TestSecureOzoneCluster {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(omKeyTab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -156,7 +153,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
@@ -170,7 +167,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
@@ -189,6 +186,11 @@ public final class TestSecureOzoneCluster {
private void initSCM()
throws IOException, AuthenticationException {
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-591. Adding ASF license
header to kadm5.acl. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-591. Adding ASF license header to kadm5.acl. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c0280d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c0280d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c0280d
Branch: refs/heads/HDDS-4
Commit: b4c0280d1126eca2190dc40988695f7746429899
Parents: 602aa80
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 10 10:01:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../docker-image/docker-krb5/Dockerfile-krb5 | 1 +
.../docker-image/docker-krb5/kadm5.acl | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c0280d/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
index b5b931d..14532d4 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -28,6 +28,7 @@ RUN kdb5_util create -s -P Welcome1
RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
ADD launcher.sh .
+RUN chmod +x /opt/launcher.sh
RUN mkdir -p /data
ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c0280d/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
index 8fe9f69..f0cd660 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -1 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
*/admin@EXAMPLE.COM x
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-753. SCM security protocol
server is not starting. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-753. SCM security protocol server is not starting. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9128c665
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9128c665
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9128c665
Branch: refs/heads/HDDS-4
Commit: 9128c665ccdf26eb3152d6903b98e779585acb2d
Parents: cd470bf
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 1 15:48:52 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../scm/server/SCMSecurityProtocolServer.java | 4 +-
.../server/TestSCMSecurityProtocolServer.java | 60 ++++++++++++++++++++
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 +-
3 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index e810c54..ab29c1e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslator
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.KerberosInfo;
import org.slf4j.Logger;
@@ -60,6 +60,8 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
rpcAddress = HddsServerUtil
.getScmSecurityInetAddress(conf);
// SCM security service RPC service.
+ RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
+ ProtobufRpcEngine.class);
BlockingService secureProtoPbService =
SCMSecurityProtocolProtos.SCMSecurityProtocolService
.newReflectiveBlockingService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
new file mode 100644
index 0000000..8e7d84c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Test class for {@link SCMSecurityProtocolServer}.
+ * */
+public class TestSCMSecurityProtocolServer {
+ private SCMSecurityProtocolServer securityProtocolServer;
+ private OzoneConfiguration config;
+
+ @Rule
+ public Timeout timeout = new Timeout(1000 * 20);
+
+ @Before
+ public void setUp() throws Exception {
+ config = new OzoneConfiguration();
+ securityProtocolServer = new SCMSecurityProtocolServer(config, null);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (securityProtocolServer != null) {
+ securityProtocolServer.stop();
+ securityProtocolServer = null;
+ }
+ config = null;
+ }
+
+ @Test
+ public void testStart() {
+ securityProtocolServer.start();
+ }
+
+ @Test
+ public void testStop() {
+ securityProtocolServer.stop();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9128c665/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5791de9..904e597 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -296,7 +296,8 @@ public final class TestSecureOzoneCluster {
} catch (Exception ex) {
// Expects timeout failure from scmClient in KSM but KSM user login via
// kerberos should succeed
- Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+ Assert.assertTrue(
+ logs.getOutput().contains("Ozone Manager login successful."));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-588.
SelfSignedCertificate#generateCertificate should sign the certificate the
configured security provider. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-588. SelfSignedCertificate#generateCertificate should sign the certificate the configured security provider. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/602aa807
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/602aa807
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/602aa807
Branch: refs/heads/HDDS-4
Commit: 602aa807483464ed3b601ace2a6c379719dff653
Parents: 812c07e
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 9 00:28:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:46 2018 -0800
----------------------------------------------------------------------
.../hdds/security/x509/certificates/SelfSignedCertificate.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/602aa807/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index fef7ac3..f221246 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -103,8 +103,8 @@ public final class SelfSignedCertificate {
ContentSigner contentSigner =
- new JcaContentSignerBuilder(
- config.getSignatureAlgo()).build(key.getPrivate());
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider()).build(key.getPrivate());
// Please note: Since this is a root certificate we use "ONE" as the
// serial number. Also note that skip enforcing locale or UTC. We are
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org