You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2023/04/12 03:26:43 UTC
[ozone] branch master updated: HDDS-8286. Support SCM sub-ca certificate signed by leader SCM's sub-ca certificate (#4493)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3873d889b6 HDDS-8286. Support SCM sub-ca certificate signed by leader SCM's sub-ca certificate (#4493)
3873d889b6 is described below
commit 3873d889b63b3163609a1171270f39d1a693d0b3
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Wed Apr 12 11:26:38 2023 +0800
HDDS-8286. Support SCM sub-ca certificate signed by leader SCM's sub-ca certificate (#4493)
---
.../SCMSecurityProtocolServerSideTranslatorPB.java | 14 ++---
.../hdds/scm/server/SCMSecurityProtocolServer.java | 35 +++++--------
.../hdds/scm/server/StorageContainerManager.java | 12 +++--
.../compose/ozonesecure-ha/docker-compose.yaml | 53 +++++++++++++++++++
.../main/compose/ozonesecure-ha/docker-config-scm4 | 18 +++++++
.../dist/src/main/compose/ozonesecure-ha/test.sh | 12 +++++
hadoop-ozone/dist/src/main/compose/testlib.sh | 27 ++++++++++
.../src/main/smoketest/scmha/primordial-scm.robot | 60 ++++++++++++++++++++++
.../main/smoketest/scmha/scm-leader-transfer.robot | 2 +-
9 files changed, 196 insertions(+), 37 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
index 182e4e5887..736aef15a0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -86,14 +86,10 @@ public class SCMSecurityProtocolServerSideTranslatorPB
@Override
public SCMSecurityResponse submitRequest(RpcController controller,
SCMSecurityRequest request) throws ServiceException {
- // For request type GetSCMCertificate we don't need leader check. As
- // primary SCM may not be leader SCM.
- if (!request.getCmdType().equals(GetSCMCertificate)) {
- if (!scm.checkLeader()) {
- RatisUtil.checkRatisException(
- scm.getScmHAManager().getRatisServer().triggerNotLeaderException(),
- scm.getSecurityProtocolRpcPort(), scm.getScmId());
- }
+ if (!scm.checkLeader()) {
+ RatisUtil.checkRatisException(
+ scm.getScmHAManager().getRatisServer().triggerNotLeaderException(),
+ scm.getSecurityProtocolRpcPort(), scm.getScmId());
}
return dispatcher.processRequest(request, this::processRequest,
request.getCmdType(), request.getTraceID());
@@ -250,7 +246,7 @@ public class SCMSecurityProtocolServerSideTranslatorPB
SCMGetSCMCertRequestProto request)
throws IOException {
- if (!scm.getScmStorageConfig().checkPrimarySCMIdInitialized()) {
+ if (!scm.getScmStorageConfig().isSCMHAEnabled()) {
throw createNotHAException();
}
String certificate = impl.getSCMCertificate(request.getScmDetails(),
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index f394daa82e..2a99415573 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND;
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CA_CERT_FAILED;
import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CERTIFICATE_FAILED;
-import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.NOT_A_PRIMARY_SCM;
import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateApprover.ApprovalType.KERBEROS_TRUSTED;
/**
@@ -191,28 +190,18 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails,
String certSignReq) throws IOException {
Objects.requireNonNull(scmNodeDetails);
- String primaryScmId =
- storageContainerManager.getScmStorageConfig().getPrimaryScmNodeId();
-
- if (primaryScmId != null &&
- primaryScmId.equals(storageContainerManager.getScmId())) {
- LOGGER.info("Processing CSR for scm {}, nodeId: {}",
- scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId());
-
- // Check clusterID
- if (!storageContainerManager.getClusterId().equals(
- scmNodeDetails.getClusterId())) {
- throw new IOException("SCM ClusterId mismatch. Peer SCM ClusterId " +
- scmNodeDetails.getClusterId() + ", primary SCM ClusterId "
- + storageContainerManager.getClusterId());
- }
-
- return getEncodedCertToString(certSignReq, NodeType.SCM);
- } else {
- throw new SCMSecurityException("Get SCM Certificate can be run only " +
- "primary SCM", NOT_A_PRIMARY_SCM);
+ // Check clusterID
+ if (!storageContainerManager.getClusterId().equals(
+ scmNodeDetails.getClusterId())) {
+ throw new IOException("SCM ClusterId mismatch. Peer SCM ClusterId " +
+ scmNodeDetails.getClusterId() + ", primary SCM ClusterId "
+ + storageContainerManager.getClusterId());
}
+ LOGGER.info("Processing CSR for scm {}, nodeId: {}",
+ scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId());
+
+ return getEncodedCertToString(certSignReq, NodeType.SCM);
}
/**
@@ -225,9 +214,9 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
private String getEncodedCertToString(String certSignReq, NodeType nodeType)
throws IOException {
Future<CertPath> future;
- if (nodeType == NodeType.SCM) {
+ if (nodeType == NodeType.SCM && rootCertificateServer != null) {
future = rootCertificateServer.requestCertificate(certSignReq,
- KERBEROS_TRUSTED, nodeType);
+ KERBEROS_TRUSTED, nodeType);
} else {
future = scmCertificateServer.requestCertificate(certSignReq,
KERBEROS_TRUSTED, nodeType);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 3761185211..b3696c57b1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -818,7 +818,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
} else {
scmCertificateServer = new DefaultCAServer(subject,
scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(),
- certificateStore, new DefaultProfile(),
+ certificateStore, new DefaultCAProfile(),
scmCertificateClient.getComponentName());
// INTERMEDIARY_CA which issues certs to DN and OM.
scmCertificateServer.init(new SecurityConfig(configuration),
@@ -854,8 +854,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
// as for SCM CA is root-CA.
securityProtocolServer = new SCMSecurityProtocolServer(conf,
rootCertificateServer, scmCertificateServer,
- scmCertificateClient != null ?
- scmCertificateClient.getCACertificate() : null, this);
+ scmCertificateClient == null ? null :
+ scmCertificateClient.getRootCACertificate() != null ?
+ scmCertificateClient.getRootCACertificate() :
+ scmCertificateClient.getCACertificate(), this);
if (securityConfig.isContainerTokenEnabled()) {
containerTokenMgr = createContainerTokenSecretManager(configuration);
@@ -1103,6 +1105,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
+ "{}, self id {} " + "Ignoring it.", primordialSCM, selfNodeId);
return true;
}
+
+ loginAsSCMUserIfSecurityEnabled(scmhaNodeDetails, conf);
+
final String persistedClusterId = scmStorageConfig.getClusterID();
StorageState state = scmStorageConfig.getState();
if (state == StorageState.INITIALIZED && conf
@@ -1119,7 +1124,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return true;
}
- loginAsSCMUserIfSecurityEnabled(scmhaNodeDetails, conf);
// The node here will try to fetch the cluster id from any of existing
// running SCM instances.
OzoneConfiguration config =
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
index 03c010e9a2..a13c0b2f50 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
@@ -301,6 +301,59 @@ services:
networks:
ozone_net:
ipv4_address: 172.25.0.118
+ scm4.org:
+ image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+ hostname: scm4.org
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 10004:9876
+ - 10006:9860
+ env_file:
+ - docker-config
+ - docker-config-scm4
+ environment:
+ ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION
+ OZONE_OPTS:
+ command: [ "/opt/hadoop/bin/ozone","scm" ]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.120
+ profiles: ["scm4.org"]
+ datanode4:
+ image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 10008:9999
+ command: [ "/opt/hadoop/bin/ozone","datanode" ]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ - "scm4.org: 172.25.0.120"
+ - "recon: 172.25.0.115"
+ env_file:
+ - docker-config
+ - docker-config-scm4
+ environment:
+ WAITFOR: scm4.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.121
+ profiles: [ "datanode4" ]
recon:
image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
hostname: recon
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-scm4 b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-scm4
new file mode 100644
index 0000000000..39d7e8b583
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-scm4
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3,scm4
+OZONE-SITE.XML_ozone.scm.address.scmservice.scm4=scm4.org
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
index d082206f6e..0b39ffb9c1 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
@@ -58,6 +58,18 @@ execute_robot_test s3g httpfs
export SCM=scm2.org
execute_robot_test s3g admincli
+# bootstrap new SCM4
+docker-compose up -d scm4.org
+wait_for_port scm4.org 9894 120
+execute_robot_test scm4.org kinit.robot
+wait_for_execute_command scm4.org 120 "ozone admin scm roles | grep scm4.org"
+execute_robot_test scm4.org scmha/primordial-scm.robot
+
+# add new datanode4
+docker-compose up -d datanode4
+wait_for_port datanode4 9856 60
+wait_for_execute_command scm4.org 60 "ozone admin datanode list | grep datanode4"
+
stop_docker_env
generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 3026c21f75..87d1773a82 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -315,6 +315,33 @@ wait_for_port(){
return 1
}
+## @description wait for the stat to be ready
+## @param The container ID
+## @param The maximum time to wait in seconds
+## @param The command line to be executed
+wait_for_execute_command(){
+ local container=$1
+ local timeout=$2
+ local command=$3
+
+ #Reset the timer
+ SECONDS=0
+
+ while [[ $SECONDS -lt $timeout ]]; do
+ set +e
+ docker-compose exec -T $container bash -c '$command'
+ status=$?
+ set -e
+ if [ $status -eq 0 ] ; then
+ echo "$command succeed"
+ return;
+ fi
+ echo "$command hasn't succeed yet"
+ sleep 1
+ done
+ echo "Timed out waiting on $command to be successful"
+ return 1
+}
## @description Stops a docker-compose based test environment (with saving the logs)
stop_docker_env(){
diff --git a/hadoop-ozone/dist/src/main/smoketest/scmha/primordial-scm.robot b/hadoop-ozone/dist/src/main/smoketest/scmha/primordial-scm.robot
new file mode 100644
index 0000000000..879940093a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/scmha/primordial-scm.robot
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoketest ozone cluster startup
+Library OperatingSystem
+Library BuiltIn
+Resource ../commonlib.robot
+Test Timeout 5 minutes
+
+*** Variables ***
+
+** Keywords ***
+Get SCM Node Count
+ ${result} = Execute ozone admin scm roles --service-id=scmservice
+ LOG ${result}
+ ${scm_count} = Get Line Count ${result}
+ [return] ${scm_count}
+
+*** Test Cases ***
+Verify SCM Count
+ ${scm_count} = Get SCM Node Count
+ LOG SCM Instance Count: ${scm_count}
+ ${scm_count} = Convert To String ${scm_count}
+ Should be Equal 4 ${scm_count}
+
+Transfer Leader to SCM4
+ ${result} = Execute ozone admin scm roles --service-id=scmservice
+ LOG ${result}
+ ${scm4_line} = Get Lines Containing String ${result} scm4.org
+ ${scm4_split} = Split String ${scm4_line} :
+ ${scm4_uuid} = Strip String ${scm4_split[3]}
+
+ ${result} = Execute ozone admin scm transfer --service-id=scmservice -n ${scm4_uuid}
+ LOG ${result}
+ Should Contain ${result} Transfer leadership successfully
+
+Verify SCM4 Certificate
+ ${root_ca_output} = Execute ozone admin cert list -c 1
+ ${root_ca_cert} = Get Lines Containing String ${root_ca_output} scm1.org
+ ${root_ca_cert_split} = Split String ${root_ca_cert}
+ ${root_ca_cert_subject} = Strip String ${root_ca_cert_split[3]}
+
+ ${output} = Execute ozone admin cert list
+ ${scm4_cert} = Get Lines Containing String ${output} scm4.org
+ ${scm4_cert_split} = Split String ${scm4_cert}
+ ${scm4_cert_issuer} = Strip String ${scm4_cert_split[4]}
+ Should not Be Equal As Strings ${scm4_cert_issuer} ${root_ca_cert_subject}
diff --git a/hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot b/hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot
index 80be127ef0..cf38de1597 100644
--- a/hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/scmha/scm-leader-transfer.robot
@@ -30,7 +30,7 @@ Get SCM Leader Node
Should Contain ${result} FOLLOWER 2
${scmLine} = Get Lines Containing String ${result} LEADER
${splits} = Split String ${scmLine} :
- ${leaderSCM} = Strip String ${splits[4]}
+ ${leaderSCM} = Strip String ${splits[3]}
LOG Leader SCM: ${leaderSCM}
[return] ${leaderSCM}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org