You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by zi...@apache.org on 2024/06/11 07:17:42 UTC
(ozone) branch master updated: HDDS-10889. Remove certificate revocation related code. (#6725)
This is an automated email from the ASF dual-hosted git repository.
zitadombi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 65800dbc52 HDDS-10889. Remove certificate revocation related code. (#6725)
65800dbc52 is described below
commit 65800dbc5294b3fedc72a35bf587b69c1d287444
Author: Istvan Fajth <fa...@gmail.com>
AuthorDate: Tue Jun 11 09:17:37 2024 +0200
HDDS-10889. Remove certificate revocation related code. (#6725)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 -
.../hadoop/hdds/scm/exceptions/SCMException.java | 6 +
.../hadoop/hdds/security/SecurityConfig.java | 15 -
.../hadoop/hdds/security/x509/crl/CRLStatus.java | 87 ------
.../hdds/security/x509/crl/package-info.java | 24 --
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 -
.../common/src/main/resources/ozone-default.xml | 8 -
.../hdds/datanode/metadata/CRLDBDefinition.java | 100 -------
.../hdds/datanode/metadata/DatanodeCRLStore.java | 84 ------
.../datanode/metadata/DatanodeCRLStoreImpl.java | 128 ---------
.../hdds/datanode/metadata/package-info.java | 22 --
.../apache/hadoop/ozone/HddsDatanodeService.java | 21 +-
.../common/report/CRLStatusReportPublisher.java | 82 ------
.../common/report/ReportPublisherFactory.java | 2 -
.../common/statemachine/DatanodeStateMachine.java | 12 +-
.../common/statemachine/StateContext.java | 13 -
.../metadata/TestDatanodeCRLStoreImpl.java | 117 --------
.../hdds/datanode/metadata/package-info.java | 22 --
.../hadoop/ozone/TestHddsDatanodeService.java | 24 +-
.../common/report/TestReportPublisher.java | 74 -----
.../common/report/TestReportPublisherFactory.java | 12 -
.../common/statemachine/TestStateContext.java | 11 +-
.../content/concept/StorageContainerManager.md | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 31 +--
.../SCMSecurityProtocolClientSideTranslatorPB.java | 61 +----
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 46 ----
.../scm/update/client/CRLClientUpdateHandler.java | 202 --------------
.../hadoop/hdds/scm/update/client/CRLStore.java | 34 ---
.../hdds/scm/update/client/ClientCRLStore.java | 98 -------
.../scm/update/client/ClientUpdateHandler.java | 29 --
.../client/SCMUpdateClientConfiguration.java | 53 ----
.../update/client/SCMUpdateServiceGrpcClient.java | 219 ---------------
.../scm/update/client/UpdateServiceConfig.java | 43 ---
.../hdds/scm/update/client/package-info.java | 22 --
.../hdds/scm/update/server/CRLClientInfo.java | 56 ----
.../scm/update/server/SCMUpdateClientInfo.java | 67 -----
.../hdds/scm/update/server/package-info.java | 22 --
.../x509/certificate/authority/CRLApprover.java | 43 ---
.../certificate/authority/CertificateServer.java | 40 +--
.../certificate/authority/CertificateStore.java | 93 +------
.../certificate/authority/DefaultCAServer.java | 61 +----
.../certificate/authority/DefaultCRLApprover.java | 58 ----
.../hadoop/hdds/security/x509/crl/CRLCodec.java | 220 ---------------
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 226 ---------------
.../hdds/security/x509/crl/package-info.java | 23 --
.../x509/certificate/authority/MockCAStore.java | 56 +---
.../certificate/authority/TestDefaultCAServer.java | 55 ----
.../x509/certificate/utils/TestCRLCodec.java | 288 --------------------
.../src/main/proto/ScmServerProtocol.proto | 3 +
.../src/main/proto/ScmServerSecurityProtocol.proto | 20 ++
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 -
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 -
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 45 ---
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 77 ------
.../SCMSecurityProtocolServerSideTranslatorPB.java | 60 +---
.../hdds/scm/security/CRLStatusReportHandler.java | 87 ------
.../hdds/scm/security/RootCARotationManager.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 205 +-------------
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 14 -
.../hdds/scm/server/SCMSecurityProtocolServer.java | 71 +----
.../hdds/scm/server/StorageContainerManager.java | 31 +--
.../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 ----
.../scm/update/server/SCMCRLUpdateHandler.java | 142 ----------
.../scm/update/server/SCMUpdateClientManager.java | 149 ----------
.../hdds/scm/update/server/SCMUpdateHandler.java | 53 ----
.../update/server/SCMUpdateServiceGrpcServer.java | 100 -------
.../scm/update/server/SCMUpdateServiceImpl.java | 119 --------
.../hdds/scm/update/server/package-info.java | 22 --
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 16 --
.../scm/security/TestCRLStatusReportHandler.java | 137 ----------
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 198 +-------------
.../hdds/scm/update/server/MockCRLStore.java | 140 ----------
.../server/TestSCMUpdateServiceGrpcServer.java | 302 ---------------------
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 15 +-
.../dist/src/main/smoketest/admincli/cert.robot | 4 +-
.../src/main/smoketest/security/admin-cert.robot | 5 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 -
.../apache/hadoop/ozone/TestDelegationToken.java | 2 -
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
80 files changed, 107 insertions(+), 5014 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 609baeeaf7..40eb8d1f85 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -38,10 +38,6 @@ public final class HddsConfigKeys {
"hdds.container.report.interval";
public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
"60m";
- public static final String HDDS_CRL_STATUS_REPORT_INTERVAL =
- "hdds.crl.status.report.interval";
- public static final String HDDS_CRL_STATUS_REPORT_INTERVAL_DEFAULT =
- "60s";
public static final String HDDS_PIPELINE_REPORT_INTERVAL =
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
@@ -184,9 +180,6 @@ public final class HddsConfigKeys {
public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
public static final String HDDS_X509_FILE_NAME_DEFAULT = "certificate.crt";
- public static final String HDDS_X509_CRL_NAME = "hdds.x509.CRL.name";
- public static final String HDDS_X509_CRL_NAME_DEFAULT = "scm.crl";
-
/**
* Default duration of certificates issued by SCM CA. The formats accepted are
* based on the ISO-8601 duration format PnDTnHnMn.nS Default value is 5 years
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
index 1cebd3296e..b1dffa5e51 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
@@ -126,12 +126,18 @@ public class SCMException extends IOException {
FAILED_TO_INIT_PIPELINE_CHOOSE_POLICY,
FAILED_TO_INIT_LEADER_CHOOSE_POLICY,
SCM_NOT_LEADER,
+ // Unused, revocation code was removed. Re-use with care when implementing revocation support.
+ // (To keep ordinals, we should not delete this from the enum.)
FAILED_TO_REVOKE_CERTIFICATES,
PIPELINE_NOT_FOUND,
UNKNOWN_PIPELINE_STATE,
CONTAINER_NOT_FOUND,
CONTAINER_REPLICA_NOT_FOUND,
+ // Unused, revocation code was removed. Re-use with care when implementing revocation support.
+ // (To keep ordinals, we should not delete this from the enum.)
FAILED_TO_CONNECT_TO_CRL_SERVICE,
+ // Unused, revocation code was removed. Re-use with care when implementing revocation support.
+ // (To keep ordinals, we should not delete this from the enum.)
FAILED_TO_ADD_CRL_CLIENT,
INVALID_PIPELINE_STATE,
DUPLICATED_PIPELINE_ID,
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
index 49af626dbe..a3c3b7061f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
@@ -78,8 +78,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CRL_NAME;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CRL_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME;
@@ -126,7 +124,6 @@ public class SecurityConfig {
private final Duration defaultCertDuration;
private final Duration renewalGracePeriod;
private final boolean isSecurityEnabled;
- private final String crlName;
private final boolean grpcTlsUseTestCert;
private final String externalRootCaPublicKeyPath;
private final String externalRootCaPrivateKeyPath;
@@ -263,9 +260,6 @@ public class SecurityConfig {
HDDS_X509_ROOTCA_PRIVATE_KEY_FILE,
HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT);
- this.crlName = configuration.get(HDDS_X509_CRL_NAME,
- HDDS_X509_CRL_NAME_DEFAULT);
-
this.grpcSSLProvider = SslProvider.valueOf(
configuration.get(HDDS_GRPC_TLS_PROVIDER,
HDDS_GRPC_TLS_PROVIDER_DEFAULT));
@@ -357,15 +351,6 @@ public class SecurityConfig {
}
}
- /**
- * Returns the CRL Name.
- *
- * @return String.
- */
- public String getCrlName() {
- return crlName;
- }
-
/**
* Returns true if security is enabled for OzoneCluster. This is determined by
* value of OZONE_SECURITY_ENABLED_KEY.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLStatus.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLStatus.java
deleted file mode 100644
index 099d050772..0000000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLStatus.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.crl;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-import java.util.List;
-
-/**
- * Class that contains the CRL status.
- */
-public class CRLStatus {
- private final long receivedCRLId;
- private final List<Long> pendingCRLIds;
-
- /**
- * Constructs a CRL Status.
- * @param receivedCRLId The last received CRL id.
- * @param pendingCRLIds A list of CRL Ids that are pending processing.
- */
- public CRLStatus(long receivedCRLId, List<Long> pendingCRLIds) {
- this.receivedCRLId = receivedCRLId;
- this.pendingCRLIds = pendingCRLIds;
- }
-
-
- public long getReceivedCRLId() {
- return receivedCRLId;
- }
-
- public List<Long> getPendingCRLIds() {
- return pendingCRLIds;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof CRLStatus)) {
- return false;
- }
-
- CRLStatus that = (CRLStatus) obj;
-
- if (this.receivedCRLId != that.getReceivedCRLId()) {
- return false;
- }
- if (this.pendingCRLIds.size() != that.getPendingCRLIds().size()) {
- return false;
- }
-
- return CollectionUtils.isEqualCollection(this.pendingCRLIds,
- that.getPendingCRLIds());
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(81, 145)
- .append(receivedCRLId)
- .append(pendingCRLIds)
- .toHashCode();
- }
-
- @Override
- public String toString() {
- return "CRLStatus{" +
- ", receivedCRLId=" + receivedCRLId +
- ", pendingCRLIds=" + StringUtils.join(pendingCRLIds, ",") +
- '}';
- }
-}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java
deleted file mode 100644
index 4fa5c7679c..0000000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-
-/**
- * This package contains common routines used in creating an x509 CRL.
- */
-package org.apache.hadoop.hdds.security.x509.crl;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 4e3f7e01cb..79ba9bbacf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -127,10 +127,7 @@ public final class OzoneConsts {
*/
public static final String CONTAINER_DB_SUFFIX = "container.db";
public static final String PIPELINE_DB_SUFFIX = "pipeline.db";
- public static final String CRL_DB_SUFFIX = "crl.db";
public static final String DN_CONTAINER_DB = "-dn-" + CONTAINER_DB_SUFFIX;
- public static final String DN_CRL_DB = "dn-" + CRL_DB_SUFFIX;
- public static final String CRL_DB_DIRECTORY_NAME = "crl";
public static final String OM_DB_NAME = "om.db";
public static final String SCM_DB_NAME = "scm.db";
public static final String OM_DB_BACKUP_PREFIX = "om.db.backup.";
@@ -452,9 +449,6 @@ public final class OzoneConsts {
public static final String SCM_DUMMY_NODEID = "scmNodeId";
public static final String SCM_DUMMY_SERVICE_ID = "scmServiceId";
- // CRL Sequence Id
- public static final String CRL_SEQUENCE_ID_KEY = "CRL_SEQUENCE_ID";
-
public static final String SCM_CA_PATH = "ca";
public static final String SCM_CA_CERT_STORAGE_DIR = "scm";
public static final String SCM_SUB_CA_PATH = "sub-ca";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 1c23c7c8ec..9b864a65c7 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -369,14 +369,6 @@
datanode periodically send container report to SCM. Unit could be
defined with postfix (ns,ms,s,m,h,d)</description>
</property>
- <property>
- <name>hdds.crl.status.report.interval</name>
- <value>60000ms</value>
- <tag>OZONE, SECURITY, MANAGEMENT</tag>
- <description>Time interval of the datanode to send CRL status report. Each
- datanode periodically sends CRL status report to SCM. Unit could be
- defined with postfix (ns,ms,s,m,h,d)</description>
- </property>
<property>
<name>hdds.pipeline.report.interval</name>
<value>60000ms</value>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
deleted file mode 100644
index 970f682d47..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
-import org.apache.hadoop.hdds.utils.db.DBDefinition;
-import org.apache.hadoop.hdds.utils.db.LongCodec;
-import org.apache.hadoop.hdds.utils.db.StringCodec;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.File;
-import java.util.Map;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-
-/**
- * Class defines the structure and types of the crl.db.
- */
-public class CRLDBDefinition extends DBDefinition.WithMap {
-
- public static final DBColumnFamilyDefinition<Long, CRLInfo> PENDING_CRLS =
- new DBColumnFamilyDefinition<>(
- "pendingCrls",
- Long.class,
- LongCodec.get(),
- CRLInfo.class,
- CRLInfo.getCodec());
-
- public static final DBColumnFamilyDefinition<String, Long>
- CRL_SEQUENCE_ID =
- new DBColumnFamilyDefinition<>(
- "crlSequenceId",
- String.class,
- StringCodec.get(),
- Long.class,
- LongCodec.get());
-
- private static final Map<String, DBColumnFamilyDefinition<?, ?>>
- COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
- PENDING_CRLS, CRL_SEQUENCE_ID);
-
- public CRLDBDefinition() {
- // TODO: change it to singleton
- super(COLUMN_FAMILIES);
- }
-
- @Override
- public String getName() {
- return OzoneConsts.DN_CRL_DB;
- }
-
- @Override
- public String getLocationConfigKey() {
- throw new UnsupportedOperationException(
- "No location config key available for datanode databases.");
- }
-
- @Override
- public File getDBLocation(ConfigurationSource conf) {
- // Please Note: To make it easy for our customers we will attempt to read
- // HDDS metadata dir and if that is not set, we will use Ozone directory.
- String metadataDir = conf.get(HDDS_METADATA_DIR_NAME,
- conf.get(OZONE_METADATA_DIRS));
- Preconditions.checkNotNull(metadataDir, "Metadata directory can't be"
- + " null. Please check configs.");
-
- // create directories in the path if they do not already exist
- HddsUtils.createDir(metadataDir
- + File.separator
- + DNCertificateClient.COMPONENT_NAME);
-
- return HddsUtils.createDir(metadataDir
- + File.separator
- + DNCertificateClient.COMPONENT_NAME
- + File.separator
- + OzoneConsts.CRL_DB_DIRECTORY_NAME);
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStore.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStore.java
deleted file mode 100644
index 0dc606f860..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStore.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.utils.db.DBStore;
-import org.apache.hadoop.hdds.utils.db.Table;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Generic interface for data stores for Datanode.
- * This is similar to the OMMetadataStore class,
- * where we write classes into some underlying storage system.
- */
-public interface DatanodeCRLStore {
-
- /**
- * Start metadata manager.
- *
- * @param configuration - Configuration
- * @throws IOException - Unable to start metadata store.
- */
- void start(OzoneConfiguration configuration) throws IOException;
-
- /**
- * Stop metadata manager.
- */
- void stop() throws Exception;
-
- /**
- * Get metadata store.
- *
- * @return metadata store.
- */
- @VisibleForTesting
- DBStore getStore();
-
- /**
- * A table to store the latest processed CRL sequence Id.
- * @return Table
- */
- Table<String, Long> getCRLSequenceIdTable();
-
- /**
- * A table to store all the pending CRLs for future revocation of
- * certificates.
- * @return Table
- */
- Table<Long, CRLInfo> getPendingCRLsTable();
-
- /**
- * Returns the latest processed CRL Sequence ID.
- * @return CRL Sequence ID.
- * @throws IOException on error.
- */
- Long getLatestCRLSequenceID() throws IOException;
-
- /**
- * Return a list of CRLs that are pending revocation.
- * @return a list of CRLInfo.
- * @throws IOException on error.
- */
- List<CRLInfo> getPendingCRLs() throws IOException;
-
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStoreImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStoreImpl.java
deleted file mode 100644
index 0926a79295..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStoreImpl.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.utils.db.DBStore;
-import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
-import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.hdds.datanode.metadata.CRLDBDefinition.CRL_SEQUENCE_ID;
-import static org.apache.hadoop.hdds.datanode.metadata.CRLDBDefinition.PENDING_CRLS;
-import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
-
-/**
- * A RocksDB based implementation of the Datanode CRL Store.
- */
-public class DatanodeCRLStoreImpl implements DatanodeCRLStore {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(DatanodeCRLStore.class);
- private DBStore store;
- private Table<String, Long> crlSequenceIdTable;
- private Table<Long, CRLInfo> pendingCRLsTable;
-
- /**
- * Constructs the metadata store and starts the DB Services.
- *
- * @param configuration - Ozone Configuration.
- * @throws IOException - on Failure.
- */
- public DatanodeCRLStoreImpl(OzoneConfiguration configuration)
- throws IOException {
- start(configuration);
- }
-
- @Override
- public void start(OzoneConfiguration configuration) throws IOException {
- if (this.store == null) {
- this.store = DBStoreBuilder.createDBStore(configuration,
- new CRLDBDefinition());
-
- crlSequenceIdTable = CRL_SEQUENCE_ID.getTable(store);
- checkTableStatus(crlSequenceIdTable, CRL_SEQUENCE_ID.getName());
-
- pendingCRLsTable = PENDING_CRLS.getTable(store);
- checkTableStatus(pendingCRLsTable, PENDING_CRLS.getName());
- }
- }
-
- @Override
- public void stop() throws Exception {
- if (store != null) {
- store.close();
- store = null;
- }
- }
-
- @Override
- public DBStore getStore() {
- return this.store;
- }
-
- @Override
- public Table<String, Long> getCRLSequenceIdTable() {
- return crlSequenceIdTable;
- }
-
- @Override
- public Table<Long, CRLInfo> getPendingCRLsTable() {
- return pendingCRLsTable;
- }
-
- @Override
- public Long getLatestCRLSequenceID() throws IOException {
- Long sequenceId = crlSequenceIdTable.get(CRL_SEQUENCE_ID_KEY);
- // If the CRL_SEQUENCE_ID_KEY does not exist in DB return 0
- if (sequenceId == null) {
- return 0L;
- }
- return sequenceId;
- }
-
- @Override
- public List<CRLInfo> getPendingCRLs() throws IOException {
- try (TableIterator<Long, ? extends Table.KeyValue<Long, CRLInfo>> iter =
- pendingCRLsTable.iterator()) {
- List<CRLInfo> pendingCRLs = new ArrayList<>();
- while (iter.hasNext()) {
- pendingCRLs.add(iter.next().getValue());
- }
- return pendingCRLs;
- }
- }
-
- private void checkTableStatus(Table table, String name) throws IOException {
- String logMessage = "Unable to get a reference to %s table. Cannot " +
- "continue.";
- String errMsg = "Inconsistent DB state, Table - %s. Please check the logs" +
- "for more info.";
- if (table == null) {
- LOG.error(String.format(logMessage, name));
- throw new IOException(String.format(errMsg, name));
- }
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java
deleted file mode 100644
index 4ca917c474..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains classes for Datanode metadata definitions.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index f59622cb0f..55125e5fb3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -38,8 +38,6 @@ import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
-import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
-import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.SecretKeyProtocol;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
@@ -122,7 +120,6 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
private DNMXBeanImpl serviceRuntimeInfo =
new DNMXBeanImpl(HddsVersionInfo.HDDS_VERSION_INFO) { };
private ObjectName dnInfoBeanName;
- private DatanodeCRLStore dnCRLStore;
private HddsDatanodeClientProtocolServer clientProtocolServer;
private OzoneAdmins admins;
private ReconfigurationHandler reconfigurationHandler;
@@ -272,9 +269,6 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
layoutStorage.initialize();
}
- // initialize datanode CRL store
- dnCRLStore = new DatanodeCRLStoreImpl(conf);
-
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
dnCertClient = initializeCertificateClient(dnCertClient);
@@ -297,7 +291,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
this::reconfigReplicationStreamsLimit);
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf,
- dnCertClient, secretKeyClient, this::terminateDatanode, dnCRLStore,
+ dnCertClient, secretKeyClient, this::terminateDatanode,
reconfigurationHandler);
try {
httpServer = new HddsDatanodeHttpServer(conf);
@@ -504,11 +498,6 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
return clientProtocolServer;
}
- @VisibleForTesting
- public DatanodeCRLStore getCRLStore() {
- return dnCRLStore;
- }
-
public void join() {
if (datanodeStateMachine != null) {
try {
@@ -560,14 +549,6 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
getClientProtocolServer().stop();
}
unregisterMXBean();
- // stop dn crl store
- try {
- if (dnCRLStore != null) {
- dnCRLStore.stop();
- }
- } catch (Exception ex) {
- LOG.error("Datanode CRL store stop failed", ex);
- }
RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList);
if (secretKeyClient != null) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CRLStatusReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CRLStatusReportPublisher.java
deleted file mode 100644
index 9b7cf9aec3..0000000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CRLStatusReportPublisher.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.report;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CRL_STATUS_REPORT_INTERVAL;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CRL_STATUS_REPORT_INTERVAL_DEFAULT;
-
-/**
- * Publishes CRLStatusReport which will be sent to SCM as part of heartbeat.
- * CRLStatusReport consist of the following information:
- * - receivedCRLId : The latest processed CRL Sequence ID.
- * - pendingCRLIds : The list of CRL IDs that are still pending in the
- * queue to be processed in the future.
- */
-public class CRLStatusReportPublisher extends
- ReportPublisher<CRLStatusReport> {
-
- private Long crlStatusReportInterval = null;
-
- @Override
- protected long getReportFrequency() {
- if (crlStatusReportInterval == null) {
- crlStatusReportInterval = getConf().getTimeDuration(
- HDDS_CRL_STATUS_REPORT_INTERVAL,
- HDDS_CRL_STATUS_REPORT_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
-
- long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
- getConf());
-
- Preconditions.checkState(
- heartbeatFrequency <= crlStatusReportInterval,
- HDDS_CRL_STATUS_REPORT_INTERVAL +
- " cannot be configured lower than heartbeat frequency.");
- }
- return crlStatusReportInterval;
- }
-
- @Override
- protected CRLStatusReport getReport() throws IOException {
-
- CRLStatusReport.Builder builder = CRLStatusReport.newBuilder();
-
- DatanodeCRLStore dnCRLStore = this.getContext().getParent().getDnCRLStore();
-
- builder.setReceivedCrlId(dnCRLStore.getLatestCRLSequenceID());
- if (dnCRLStore.getPendingCRLs().size() > 0) {
- List<Long> pendingCRLIds =
- dnCRLStore.getPendingCRLs().stream().map(CRLInfo::getCrlSequenceID)
- .collect(Collectors.toList());
- builder.addAllPendingCrlIds(pendingCRLIds);
- }
-
- return builder.build();
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index 3be1b5e077..3fd56f3405 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
@@ -54,7 +53,6 @@ public class ReportPublisherFactory {
CommandStatusReportPublisher.class);
report2publisher.put(PipelineReportsProto.class,
PipelineReportPublisher.class);
- report2publisher.put(CRLStatusReport.class, CRLStatusReportPublisher.class);
}
/**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 9677144054..6046af1e0a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -33,10 +33,8 @@ import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
-import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
@@ -101,7 +99,6 @@ public class DatanodeStateMachine implements Closeable {
private final ECReconstructionCoordinator ecReconstructionCoordinator;
private StateContext context;
private final OzoneContainer container;
- private final DatanodeCRLStore dnCRLStore;
private final DatanodeDetails datanodeDetails;
private final CommandDispatcher commandDispatcher;
private final ReportManager reportManager;
@@ -146,7 +143,6 @@ public class DatanodeStateMachine implements Closeable {
CertificateClient certClient,
SecretKeyClient secretKeyClient,
HddsDatanodeStopService hddsDatanodeStopService,
- DatanodeCRLStore crlStore,
ReconfigurationHandler reconfigurationHandler)
throws IOException {
DatanodeConfiguration dnConf =
@@ -167,7 +163,6 @@ public class DatanodeStateMachine implements Closeable {
upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
VersionedDatanodeFeatures.initialize(layoutVersionManager);
- this.dnCRLStore = crlStore;
String threadNamePrefix = datanodeDetails.threadNamePrefix();
executorService = Executors.newFixedThreadPool(
getEndPointTaskThreadPoolSize(),
@@ -269,7 +264,6 @@ public class DatanodeStateMachine implements Closeable {
.addPublisherFor(ContainerReportsProto.class)
.addPublisherFor(CommandStatusReportsProto.class)
.addPublisherFor(PipelineReportsProto.class)
- .addPublisherFor(CRLStatusReport.class)
.addThreadNamePrefix(threadNamePrefix)
.build();
@@ -280,7 +274,7 @@ public class DatanodeStateMachine implements Closeable {
@VisibleForTesting
public DatanodeStateMachine(DatanodeDetails datanodeDetails,
ConfigurationSource conf) throws IOException {
- this(datanodeDetails, conf, null, null, null, null,
+ this(datanodeDetails, conf, null, null, null,
new ReconfigurationHandler("DN", (OzoneConfiguration) conf, op -> { }));
}
@@ -331,10 +325,6 @@ public class DatanodeStateMachine implements Closeable {
}
}
- public DatanodeCRLStore getDnCRLStore() {
- return dnCRLStore;
- }
-
/**
* Runs the state machine at a fixed frequency.
*/
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 6bbf8e4794..712b3f0f23 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -45,7 +45,6 @@ import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.Descriptors.Descriptor;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
@@ -95,9 +94,6 @@ public class StateContext {
@VisibleForTesting
static final String INCREMENTAL_CONTAINER_REPORT_PROTO_NAME =
IncrementalContainerReportProto.getDescriptor().getFullName();
- @VisibleForTesting
- static final String CRL_STATUS_REPORT_PROTO_NAME =
- CRLStatusReport.getDescriptor().getFullName();
static final Logger LOG =
LoggerFactory.getLogger(StateContext.class);
@@ -112,7 +108,6 @@ public class StateContext {
private final AtomicReference<Message> containerReports;
private final AtomicReference<Message> nodeReport;
private final AtomicReference<Message> pipelineReports;
- private final AtomicReference<Message> crlStatusReport;
// Incremental reports are queued in the map below
private final Map<InetSocketAddress, List<Message>>
incrementalReportsQueue;
@@ -179,7 +174,6 @@ public class StateContext {
containerReports = new AtomicReference<>();
nodeReport = new AtomicReference<>();
pipelineReports = new AtomicReference<>();
- crlStatusReport = new AtomicReference<>(); // Certificate Revocation List
endpoints = new HashSet<>();
containerActions = new HashMap<>();
pipelineActions = new HashMap<>();
@@ -204,8 +198,6 @@ public class StateContext {
type2Reports.put(NODE_REPORT_PROTO_NAME, nodeReport);
fullReportTypeList.add(PIPELINE_REPORTS_PROTO_NAME);
type2Reports.put(PIPELINE_REPORTS_PROTO_NAME, pipelineReports);
- fullReportTypeList.add(CRL_STATUS_REPORT_PROTO_NAME);
- type2Reports.put(CRL_STATUS_REPORT_PROTO_NAME, crlStatusReport);
}
/**
@@ -953,11 +945,6 @@ public class StateContext {
return pipelineReports.get();
}
- @VisibleForTesting
- public Message getCRLStatusReport() {
- return crlStatusReport.get();
- }
-
public void configureReconHeartbeatFrequency() {
reconHeartbeatFrequency.set(getReconHeartbeatInterval(conf));
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/TestDatanodeCRLStoreImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/TestDatanodeCRLStoreImpl.java
deleted file mode 100644
index f73f14f0c2..0000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/TestDatanodeCRLStoreImpl.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CRLApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCRLApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v2CRLBuilder;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import java.io.File;
-import java.security.KeyPair;
-import java.security.cert.X509Certificate;
-import java.util.Date;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-/**
- * Test class for {@link DatanodeCRLStoreImpl}.
- */
-public class TestDatanodeCRLStoreImpl {
- @TempDir
- private File testDir;
- private OzoneConfiguration conf;
- private DatanodeCRLStore dnCRLStore;
- private KeyPair keyPair;
- private CRLApprover crlApprover;
- private SecurityConfig securityConfig;
-
- @BeforeEach
- public void setUp() throws Exception {
- conf = new OzoneConfiguration();
- conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
- dnCRLStore = new DatanodeCRLStoreImpl(conf);
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- securityConfig = new SecurityConfig(conf);
- crlApprover = new DefaultCRLApprover(securityConfig,
- keyPair.getPrivate());
- }
-
- @AfterEach
- public void destroyDbStore() throws Exception {
- if (dnCRLStore.getStore() != null) {
- dnCRLStore.getStore().close();
- }
- }
- @Test
- public void testCRLStore() throws Exception {
- assertNotNull(dnCRLStore.getStore());
-
- dnCRLStore.getCRLSequenceIdTable().put(OzoneConsts.CRL_SEQUENCE_ID_KEY, 5L);
- Date now = new Date();
- X509Certificate x509Certificate = generateX509Cert();
- X509CertificateHolder caCertificateHolder =
- new X509CertificateHolder(generateX509Cert().getEncoded());
- X509v2CRLBuilder crlBuilder = new X509v2CRLBuilder(
- caCertificateHolder.getIssuer(), now);
- crlBuilder.addCRLEntry(x509Certificate.getSerialNumber(), now,
- CRLReason.lookup(CRLReason.PRIVILEGE_WITHDRAWN).getValue().intValue());
- dnCRLStore.getPendingCRLsTable().put(1L,
- new CRLInfo.Builder()
- .setCrlSequenceID(1L)
- .setCreationTimestamp(now.getTime())
- .setX509CRL(crlApprover.sign(crlBuilder))
- .build());
-
- assertEquals(5L, (long) dnCRLStore.getLatestCRLSequenceID());
- assertEquals(1L, dnCRLStore.getPendingCRLs().size());
- CRLInfo crlInfo = dnCRLStore.getPendingCRLs().get(0);
- assertEquals(1L, crlInfo.getCrlSequenceID());
- assertEquals(x509Certificate.getSerialNumber(),
- crlInfo.getX509CRL().getRevokedCertificates()
- .iterator().next().getSerialNumber());
-
- // Test that restarting the store does not affect the data already persisted
- dnCRLStore.stop();
- dnCRLStore = new DatanodeCRLStoreImpl(conf);
- assertEquals(5L, (long) dnCRLStore.getLatestCRLSequenceID());
- assertEquals(1L, dnCRLStore.getPendingCRLs().size());
- dnCRLStore.stop();
- }
-
- private X509Certificate generateX509Cert() throws Exception {
- return CertificateCodec.getX509Certificate(
- CertificateCodec.getPEMEncodedString(
- KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
- "SHA256withRSA")));
- }
-}
-
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java
deleted file mode 100644
index 43eeb2ffc8..0000000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/hdds/datanode/metadata/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * Datanode metadata Testing.
- */
-package org.apache.hadoop.hdds.datanode.metadata;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
index cc88940611..e513412e37 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -46,13 +46,11 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -106,22 +104,6 @@ public class TestHddsDatanodeService {
conf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY, volumeDir);
}
- @Test
- public void testStartup() {
- service.start(conf);
-
- assertNotNull(service.getDatanodeDetails());
- assertNotNull(service.getDatanodeDetails().getHostName());
- assertFalse(service.getDatanodeStateMachine().isDaemonStopped());
- assertNotNull(service.getCRLStore());
-
- service.stop();
- // CRL store must be stopped when the service stops
- assertNull(service.getCRLStore().getStore());
- service.join();
- service.close();
- }
-
@ParameterizedTest
@ValueSource(strings = {OzoneConsts.SCHEMA_V1,
OzoneConsts.SCHEMA_V2, OzoneConsts.SCHEMA_V3})
@@ -132,6 +114,10 @@ public class TestHddsDatanodeService {
conf.get(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED));
service.start(conf);
+ assertNotNull(service.getDatanodeDetails());
+ assertNotNull(service.getDatanodeDetails().getHostName());
+ assertFalse(service.getDatanodeStateMachine().isDaemonStopped());
+
// Get volumeSet and store volumes in temp folders
// in order to access them after service.stop()
MutableVolumeSet volumeSet = service
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index 60404fa36b..7e90893f34 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -18,39 +18,27 @@
package org.apache.hadoop.ozone.container.common.report;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsIdFactory;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import java.util.Random;
-import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -182,66 +170,4 @@ public class TestReportPublisher {
"Should publish report with 2 status objects");
executorService.shutdown();
}
-
- @Test
- public void testCRLStatusReportPublisher() throws IOException {
- StateContext dummyContext = mock(StateContext.class);
- DatanodeStateMachine dummyStateMachine =
- mock(DatanodeStateMachine.class);
- ReportPublisher publisher = new CRLStatusReportPublisher();
- DatanodeCRLStore dnCrlStore = mock(DatanodeCRLStore.class);
- when(dnCrlStore.getLatestCRLSequenceID()).thenReturn(3L);
- List<CRLInfo> pendingCRLs = new ArrayList<>();
- pendingCRLs.add(mock(CRLInfo.class));
- pendingCRLs.add(mock(CRLInfo.class));
- when(dnCrlStore.getPendingCRLs()).thenReturn(pendingCRLs);
- when(dummyStateMachine.getDnCRLStore()).thenReturn(dnCrlStore);
- when(dummyContext.getParent()).thenReturn(dummyStateMachine);
- publisher.setConf(config);
-
- ScheduledExecutorService executorService = HadoopExecutors
- .newScheduledThreadPool(1,
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("TestReportManagerThread-%d").build());
- publisher.init(dummyContext, executorService);
- Message report =
- ((CRLStatusReportPublisher) publisher).getReport();
- assertNotNull(report);
- for (Descriptors.FieldDescriptor descriptor :
- report.getDescriptorForType().getFields()) {
- if (descriptor.getNumber() ==
- CRLStatusReport.RECEIVEDCRLID_FIELD_NUMBER) {
- assertEquals(3L, report.getField(descriptor));
- }
- }
- executorService.shutdown();
- }
-
- /**
- * Get a datanode details.
- *
- * @return DatanodeDetails
- */
- private static DatanodeDetails getDatanodeDetails() {
- Random random = new Random();
- String ipAddress =
- random.nextInt(256) + "." + random.nextInt(256) + "." + random
- .nextInt(256) + "." + random.nextInt(256);
-
- DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
- DatanodeDetails.Port.Name.STANDALONE, 0);
- DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
- DatanodeDetails.Port.Name.RATIS, 0);
- DatanodeDetails.Port restPort = DatanodeDetails.newPort(
- DatanodeDetails.Port.Name.REST, 0);
- DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
- builder.setUuid(UUID.randomUUID())
- .setHostName("localhost")
- .setIpAddress(ipAddress)
- .addPort(containerPort)
- .addPort(ratisPort)
- .addPort(restPort);
- return builder.build();
- }
-
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
index 44e1389bf7..6e76f2eacc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.junit.jupiter.api.Test;
@@ -55,17 +54,6 @@ public class TestReportPublisherFactory {
assertEquals(conf, publisher.getConf());
}
- @Test
- public void testGetCRLStatusReportPublisher() {
- OzoneConfiguration conf = new OzoneConfiguration();
- ReportPublisherFactory factory = new ReportPublisherFactory(conf);
- ReportPublisher publisher = factory
- .getPublisherFor(CRLStatusReport.class);
- assertEquals(
- CRLStatusReportPublisher.class, publisher.getClass());
- assertEquals(conf, publisher.getConf());
- }
-
@Test
public void testInvalidReportPublisher() {
OzoneConfiguration conf = new OzoneConfiguration();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
index 7f2cdcc6e5..3ea478cacb 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestStateContext.java
@@ -668,8 +668,6 @@ public class TestStateContext {
totalIncrementalCount);
batchRefreshfullReports(ctx, StateContext.PIPELINE_REPORTS_PROTO_NAME,
totalIncrementalCount);
- batchRefreshfullReports(ctx,
- StateContext.CRL_STATUS_REPORT_PROTO_NAME, totalIncrementalCount);
batchAddIncrementalReport(ctx,
StateContext.INCREMENTAL_CONTAINER_REPORT_PROTO_NAME,
totalIncrementalCount);
@@ -678,12 +676,11 @@ public class TestStateContext {
expectedReportCount.put(StateContext.CONTAINER_REPORTS_PROTO_NAME, 1);
expectedReportCount.put(StateContext.NODE_REPORT_PROTO_NAME, 1);
expectedReportCount.put(StateContext.PIPELINE_REPORTS_PROTO_NAME, 1);
- expectedReportCount.put(StateContext.CRL_STATUS_REPORT_PROTO_NAME, 1);
// Should keep less or equal than maxLimit depending on other reports' size.
- // Here, the incremental container reports count must be 96
- // (100 - 4 non-incremental reports)
+ // Here, the incremental container reports count must be 97
+ // (100 - 3 non-incremental reports)
expectedReportCount.put(
- StateContext.INCREMENTAL_CONTAINER_REPORT_PROTO_NAME, 96);
+ StateContext.INCREMENTAL_CONTAINER_REPORT_PROTO_NAME, 97);
checkReportCount(ctx.getAllAvailableReportsUpToLimit(scm1, 100),
expectedReportCount);
checkReportCount(ctx.getAllAvailableReportsUpToLimit(scm2, 100),
@@ -691,7 +688,7 @@ public class TestStateContext {
expectedReportCount.clear();
expectedReportCount.put(
StateContext.INCREMENTAL_CONTAINER_REPORT_PROTO_NAME,
- totalIncrementalCount - 96);
+ totalIncrementalCount - 97);
checkReportCount(ctx.getAllAvailableReportsUpToLimit(scm1, 100),
expectedReportCount);
checkReportCount(ctx.getAllAvailableReportsUpToLimit(scm2, 100),
diff --git a/hadoop-hdds/docs/content/concept/StorageContainerManager.md b/hadoop-hdds/docs/content/concept/StorageContainerManager.md
index 860e69a77d..3b2f4d8f9d 100644
--- a/hadoop-hdds/docs/content/concept/StorageContainerManager.md
+++ b/hadoop-hdds/docs/content/concept/StorageContainerManager.md
@@ -84,7 +84,7 @@ The following data is persisted in Storage Container Manager side in a specific
* Containers are the replication units. Data is required to act in case of data under/over replicated.
* Deleted blocks
* Block data is deleted in the background. Need a list to follow the progress.
- * Valid cert, Revoked certs
+ * Valid cert
* Used by the internal Certificate Authority to authorize other Ozone services
## Notable configurations
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
index 703deec76a..cbb4f3fc2e 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmNodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeDetailsProto;
import org.apache.hadoop.hdds.scm.ScmConfig;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.security.KerberosInfo;
/**
@@ -113,11 +112,9 @@ public interface SCMSecurityProtocol {
* @param type - node type: OM/SCM/DN.
* @param startSerialId - start certificate serial id.
* @param count - max number of certificates returned in a batch.
- * @param isRevoked - whether list for revoked certs only.
* @return list of PEM encoded certificate strings.
*/
- List<String> listCertificate(HddsProtos.NodeType type, long startSerialId,
- int count, boolean isRevoked) throws IOException;
+ List<String> listCertificate(HddsProtos.NodeType type, long startSerialId, int count) throws IOException;
/**
* Get Root CA certificate.
@@ -146,32 +143,6 @@ public interface SCMSecurityProtocol {
*/
List<String> listCACertificate() throws IOException;
- /**
- * Get the CRLInfo based on the CRL Id.
- * @param crlIds - crl ids
- * @return list of CRLInfo
- * @throws IOException
- */
- List<CRLInfo> getCrls(List<Long> crlIds) throws IOException;
-
- /**
- * Get the latest CRL id.
- * @return latest CRL id.
- */
- long getLatestCrlId() throws IOException;
-
-
- /**
- * Revoke a list of certificates at specified time.
- * @param certIds - cert ids
- * @param reason - reason code: refer @org.bouncycastle.asn1.x509.CRLReason.
- * @param revocationTime - revocation time.
- * @return
- * @throws IOException
- */
- long revokeCertificates(List<String> certIds, int reason, long revocationTime)
- throws IOException;
-
/**
* Get SCM signed certificate.
*
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
index 2ab5cc1c46..a938d53c7c 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
@@ -18,15 +18,12 @@ package org.apache.hadoop.hdds.protocolPB;
import java.io.Closeable;
import java.io.IOException;
-import java.security.cert.CRLException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CRLInfoProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
@@ -37,15 +34,11 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSCM
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCACertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesRequestProto.Reason;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest.Builder;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
@@ -53,7 +46,6 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRemove
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.proxy.SCMSecurityProtocolFailoverProxyProvider;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.ProtobufHelper;
@@ -335,23 +327,20 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
}
/**
- *
- * @param role - node type: OM/SCM/DN.
- * @param startSerialId - start cert serial id.
- * @param count - max number of certificates returned in a batch.
- * @param isRevoked - whether return revoked cert only.
+ * @param role - node type: OM/SCM/DN.
+ * @param startSerialId - start cert serial id.
+ * @param count - max number of certificates returned in a batch.
* @return
* @throws IOException
*/
@Override
public List<String> listCertificate(HddsProtos.NodeType role,
- long startSerialId, int count, boolean isRevoked) throws IOException {
+ long startSerialId, int count) throws IOException {
SCMListCertificateRequestProto protoIns = SCMListCertificateRequestProto
.newBuilder()
.setRole(role)
.setStartCertId(startSerialId)
.setCount(count)
- .setIsRevoked(isRevoked)
.build();
return submitRequest(Type.ListCertificate,
builder -> builder.setListCertificateRequest(protoIns))
@@ -376,48 +365,6 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
.getListCertificateResponseProto().getCertificatesList();
}
- @Override
- public List<CRLInfo> getCrls(List<Long> crlIds) throws IOException {
- SCMGetCrlsRequestProto protoIns = SCMGetCrlsRequestProto
- .newBuilder()
- .addAllCrlId(crlIds)
- .build();
- List<CRLInfoProto> crlInfoProtoList = submitRequest(Type.GetCrls,
- builder -> builder.setGetCrlsRequest(protoIns))
- .getGetCrlsResponseProto().getCrlInfosList();
- List<CRLInfo> result = new ArrayList<>();
- for (CRLInfoProto crlProto : crlInfoProtoList) {
- try {
- CRLInfo crlInfo = CRLInfo.fromProtobuf(crlProto);
- result.add(crlInfo);
- } catch (CRLException e) {
- throw new SCMSecurityException("Fail to parse CRL info", e);
- }
- }
- return result;
- }
-
- @Override
- public long getLatestCrlId() throws IOException {
- SCMGetLatestCrlIdRequestProto protoIns = SCMGetLatestCrlIdRequestProto
- .getDefaultInstance();
- return submitRequest(Type.GetLatestCrlId,
- builder -> builder.setGetLatestCrlIdRequest(protoIns))
- .getGetLatestCrlIdResponseProto().getCrlId();
- }
-
- @Override
- public long revokeCertificates(List<String> certIds, int reason,
- long revocationTime) throws IOException {
- SCMRevokeCertificatesRequestProto req = SCMRevokeCertificatesRequestProto
- .newBuilder().addAllCertIds(certIds)
- .setReason(Reason.valueOf(reason))
- .setRevokeTime(revocationTime).build();
- return submitRequest(Type.RevokeCertificates,
- builder -> builder.setRevokeCertificatesRequest(req))
- .getRevokeCertificatesResponseProto().getCrlId();
- }
-
/**
* Return the proxy object underlying this protocol translator.
*
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
index 46b19aa090..dbe2873a23 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
@@ -27,16 +27,12 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import org.apache.hadoop.hdds.utils.DBStoreHAManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
import com.google.common.annotations.VisibleForTesting;
@@ -90,48 +86,6 @@ public interface SCMMetadataStore extends DBStoreHAManager {
*/
Table<BigInteger, X509Certificate> getValidSCMCertsTable();
- /**
- * This method is Deprecated in favor of getRevokedCertsV2Table().
- * A Table that maintains all revoked certificates until they expire.
- *
- * @return Table.
- */
- @Deprecated
- Table<BigInteger, X509Certificate> getRevokedCertsTable();
-
- /**
- * A Table that maintains all revoked certificates and the time of
- * revocation until they expire.
- *
- * @return Table.
- */
- Table<BigInteger, CertInfo> getRevokedCertsV2Table();
-
- /**
- * A table that maintains X509 Certificate Revocation Lists and its metadata.
- *
- * @return Table.
- */
- Table<Long, CRLInfo> getCRLInfoTable();
-
- /**
- * A table that maintains the last CRL SequenceId. This helps to make sure
- * that the CRL Sequence Ids are monotonically increasing.
- *
- * @return Table.
- */
- Table<String, Long> getCRLSequenceIdTable();
-
- /**
- * Returns the list of Certificates of a specific type.
- *
- * @param certType - CertType.
- * @return Iterator<X509Certificate>
- * @throws IOException on failure.
- */
- TableIterator getAllCerts(CertificateStore.CertType certType)
- throws IOException;
-
/**
* A Table that maintains all the pipeline information.
*/
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLClientUpdateHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLClientUpdateHandler.java
deleted file mode 100644
index 6723b2fd2b..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLClientUpdateHandler.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceGrpc.SCMUpdateServiceStub;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.CRLUpdateRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-import org.apache.hadoop.hdds.scm.update.server.SCMUpdateClientInfo;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * CRL client update handler that handles local CRL update and pending CRLs.
- */
-public class CRLClientUpdateHandler implements ClientUpdateHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- CRLClientUpdateHandler.class);
- private static final String NAME = "CRLClientUpdateHandler";
-
- private final SCMUpdateServiceStub updateStub;
- private final ClientCRLStore clientStore;
-
- // Used to update server about local pending crl id list
- private StreamObserver<UpdateRequest> requestObserver;
- private UUID clientUuid;
- private SCMUpdateServiceProtos.ClientId clientIdProto;
-
- // periodically process pending crls
- private ScheduledExecutorService executorService;
- private final SCMUpdateServiceGrpcClient serviceGrpcClient;
- private long crlCheckInterval;
-
- CRLClientUpdateHandler(UUID clientId,
- SCMUpdateServiceStub updateStub,
- SCMUpdateServiceGrpcClient serviceGrpcClient,
- long crlCheckInterval) {
- this.clientUuid = clientId;
- this.updateStub = updateStub;
- this.serviceGrpcClient = serviceGrpcClient;
-
- this.clientStore = serviceGrpcClient.getClientCRLStore();
- this.crlCheckInterval = crlCheckInterval;
- LOG.info("Pending CRL check interval : {}s", crlCheckInterval / 1000);
- this.executorService = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat(clientId + "-CRLUpdateHandler-%d").build());
- }
-
- public static Logger getLog() {
- return LOG;
- }
-
- @Override
- public void handleServerUpdate(UpdateResponse updateResponse) {
- SCMUpdateServiceProtos.CRLInfoProto crlInfo =
- updateResponse.getCrlUpdateResponse().getCrlInfo();
-
- long receivedCrlId = crlInfo.getCrlSequenceID();
- long localCrlId = clientStore.getLatestCrlId();
-
- LOG.debug("## Client: clientId {} clientCrlId {} receivedCrlId {}",
- clientUuid, localCrlId, receivedCrlId);
- if (localCrlId == receivedCrlId) {
- return;
- }
- // send a client update to refresh stale server
- if (localCrlId > receivedCrlId) {
- LOG.warn("Received stale crlId {} lower than client crlId {}",
- receivedCrlId, localCrlId);
- sendClientUpdate();
- return;
- }
-
- CRLInfo crl;
- try {
- crl = CRLInfo.fromCRLProto3(crlInfo);
- } catch (Exception e) {
- LOG.error("Can't parse server CRL update, skip...", e);
- return;
- }
- clientStore.onRevokeCerts(crl);
- // send client update.
- sendClientUpdate();
- }
-
- public void start() {
- // send initial update request to get a request observer handle
- UpdateRequest updateReq = getUpdateRequest();
- requestObserver = updateStub.withWaitForReady()
- .updateStatus(new StreamObserver<UpdateResponse>() {
- @Override
- public void onNext(UpdateResponse updateResponse) {
- LOG.debug("Receive server response: {}", updateResponse);
- serviceGrpcClient.incrUpdateCount();
- handleServerUpdate(updateResponse);
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOG.debug("Receive server error ", throwable);
- serviceGrpcClient.incrErrorCount();
- if (serviceGrpcClient.getIsRunning().get()) {
- // TODO: not all server error needs client restart.
- LOG.warn("Restart client on server error: ", throwable);
- serviceGrpcClient.restart();
- }
- }
-
- @Override
- public void onCompleted() {
- LOG.debug("Receive server completed");
- }
- });
- requestObserver.onNext(updateReq);
- startPendingCrlChecker();
- }
-
- public void stop() {
- stopPendingCrlCheck();
- }
-
- private void stopPendingCrlCheck() {
- executorService.shutdown();
- try {
- executorService.awaitTermination(10, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- LOG.error("InterruptedException while waiting for executor service" +
- " to shutdown", e);
- Thread.currentThread().interrupt();
- }
- }
-
- private void startPendingCrlChecker() {
- executorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- // background thread handle pending crl and update server
-
- CRLInfo crl = null;
- while ((crl = clientStore.getNextPendingCrl()) != null) {
- if (crl.shouldRevokeNow()) {
- serviceGrpcClient.incrPendingCrlRemoveCount();
- LOG.info("Time to process crlId {}", crl.getCrlSequenceID());
- clientStore.removePendingCrl(crl);
- sendClientUpdate();
- } else {
- // we are done with this pending Crl, wait for next round
- break;
- }
- }
- }
- }, 0, crlCheckInterval, TimeUnit.MILLISECONDS);
- }
-
- private void sendClientUpdate() {
- requestObserver.onNext(getUpdateRequest());
- }
-
- private UpdateRequest getUpdateRequest() {
- return UpdateRequest.newBuilder()
- .setUpdateType(SCMUpdateServiceProtos.Type.CRLUpdate)
- .setClientId(SCMUpdateClientInfo.toClientIdProto(clientUuid))
- .setCrlUpdateRequest(getCrlUpdateRequest())
- .build();
- }
-
- private CRLUpdateRequest getCrlUpdateRequest() {
- List<Long> pendingCrlIds = clientStore.getPendingCrlIds();
- return CRLUpdateRequest.newBuilder()
- .setReceivedCrlId(clientStore.getLatestCrlId())
- .addAllPendingCrlIds(pendingCrlIds)
- .build();
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLStore.java
deleted file mode 100644
index 0b98117a2d..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-
-import java.io.IOException;
-
-/**
- * CRL Store interface.
- */
-public interface CRLStore {
-
- long getLatestCrlId();
-
- CRLInfo getCRL(long crlId) throws IOException;
-
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientCRLStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientCRLStore.java
deleted file mode 100644
index 5e326ccfea..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientCRLStore.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-
-import java.io.IOException;
-import java.security.cert.X509CRL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.stream.Collectors;
-
-/**
- * In memory Client CRL store, need to integrate with Client Table.
- */
-public class ClientCRLStore implements CRLStore {
- private PriorityQueue<CRLInfo> pendingCrls;
- private List<Long> revokedCerts;
- private long localCrlId;
-
- public ClientCRLStore() {
- localCrlId = 0;
- revokedCerts = new ArrayList<>();
- pendingCrls = new PriorityQueue<>(
- new Comparator<CRLInfo>() {
- @Override
- public int compare(CRLInfo o1, CRLInfo o2) {
- return o1.getRevocationTime()
- .compareTo(o2.getRevocationTime());
- }
- });
- }
-
- @Override
- public long getLatestCrlId() {
- return localCrlId;
- }
-
- public void setLocalCrlId(long crlId) {
- localCrlId = crlId;
- }
-
-
- @Override
- public CRLInfo getCRL(long crlId) throws IOException {
- return null;
- }
-
- public void onRevokeCerts(CRLInfo crl) {
- if (crl.shouldRevokeNow()) {
- revokedCerts.addAll(getRevokedCertIds(crl.getX509CRL()));
- } else {
- pendingCrls.add(crl);
- }
- localCrlId = crl.getCrlSequenceID();
- }
-
- public List<Long> getRevokedCertIds(X509CRL crl) {
- return Collections.unmodifiableList(crl.getRevokedCertificates().stream()
- .map(cert -> cert.getSerialNumber().longValue())
- .collect(Collectors.toList()));
- }
-
- public CRLInfo getNextPendingCrl() {
- return pendingCrls.peek();
- }
-
- public void removePendingCrl(CRLInfo crl) {
- pendingCrls.remove(crl);
- revokedCerts.addAll(getRevokedCertIds(crl.getX509CRL()));
- }
-
- public List<Long> getPendingCrlIds() {
- return new ArrayList<>(pendingCrls)
- .stream().map(crl -> crl.getCrlSequenceID())
- .collect(Collectors.toList());
- }
-
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientUpdateHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientUpdateHandler.java
deleted file mode 100644
index c16e438f97..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientUpdateHandler.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-
-/**
- * Interface used by client side to handle server updates.
- */
-public interface ClientUpdateHandler {
-
- void handleServerUpdate(UpdateResponse updateResponse);
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateClientConfiguration.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateClientConfiguration.java
deleted file mode 100644
index 84fdc1f663..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateClientConfiguration.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import org.apache.hadoop.hdds.conf.Config;
-import org.apache.hadoop.hdds.conf.ConfigGroup;
-import org.apache.hadoop.hdds.conf.ConfigType;
-
-import java.time.Duration;
-
-import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
-import static org.apache.hadoop.hdds.conf.ConfigTag.SCM;
-import static org.apache.hadoop.hdds.conf.ConfigTag.SECURITY;
-
-/**
- * Configuration used by SCM CRL update client.
- */
-@ConfigGroup(prefix = "ozone.scm.update")
-public class SCMUpdateClientConfiguration {
- @Config(key = "client.crl.check.interval",
- type = ConfigType.TIME,
- defaultValue = "600s",
- tags = {SCM, OZONE, SECURITY},
- description = "The interval that the scm update service client use to" +
- "check its pending CRLs."
- )
- private long clientCrlCheckIntervalInMs =
- Duration.ofMinutes(10).toMillis();
-
- public long getClientCrlCheckInterval() {
- return clientCrlCheckIntervalInMs;
- }
-
- public void setClientCrlCheckInterval(Duration interval) {
- this.clientCrlCheckIntervalInMs = interval.toMillis();
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateServiceGrpcClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateServiceGrpcClient.java
deleted file mode 100644
index 8b96d5c0a9..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateServiceGrpcClient.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceGrpc;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.SubscribeRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.SubscribeResponse;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UnsubscribeRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateRequest;
-import org.apache.hadoop.hdds.scm.update.server.SCMUpdateClientInfo;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.ratis.thirdparty.io.grpc.Deadline;
-import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
-import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Class for SCM Update Service Grpc Client.
- */
-public class SCMUpdateServiceGrpcClient {
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMUpdateServiceGrpcClient.class);
- private static final String CLIENT_NAME = "SCMUpdateServiceGrpcClient";
-
- private ManagedChannel channel;
- private SCMUpdateServiceGrpc.SCMUpdateServiceStub updateClient;
- private SCMUpdateServiceGrpc.SCMUpdateServiceBlockingStub subscribeClient;
- private final AtomicBoolean isRunning = new AtomicBoolean(false);
-
- private UUID clientId = null;
- private StreamObserver<UpdateRequest> requestObserver;
- private CRLClientUpdateHandler handler;
- private long crlCheckInterval;
- private final String host;
- private final int port;
- private final ClientCRLStore clientCRLStore;
- private AtomicLong updateCount;
- private AtomicLong errorCount;
- private AtomicLong pendingCrlRemoveCount;
-
-
- public SCMUpdateServiceGrpcClient(final String host,
- final ConfigurationSource conf,
- ClientCRLStore clientCRLStore) {
- Preconditions.checkNotNull(conf);
- this.host = host;
- this.port = conf.getObject(UpdateServiceConfig.class).getPort();
- this.crlCheckInterval = conf.getObject(SCMUpdateClientConfiguration.class)
- .getClientCrlCheckInterval();
-
- this.clientCRLStore = clientCRLStore;
- createChannel();
- updateCount = new AtomicLong();
- errorCount = new AtomicLong();
- pendingCrlRemoveCount = new AtomicLong();
- }
-
- public void start() {
- if (!isRunning.compareAndSet(false, true)) {
- LOG.info("Ignore. already started.");
- return;
- }
-
- LOG.info("{}: starting...", CLIENT_NAME);
- if (channel == null) {
- createChannel();
- }
- clientId = subScribeClient();
- assert (clientId != null);
-
- // start background thread processing pending crl ids.
- handler = new CRLClientUpdateHandler(clientId, updateClient,
- this, crlCheckInterval);
- handler.start();
-
- LOG.info("{}: started.", CLIENT_NAME);
- }
-
- public void incrUpdateCount() {
- updateCount.incrementAndGet();
- }
-
- public void incrErrorCount() {
- errorCount.incrementAndGet();
- }
-
- public void incrPendingCrlRemoveCount() {
- pendingCrlRemoveCount.incrementAndGet();
- }
-
- @VisibleForTesting
- public long getUpdateCount() {
- return updateCount.get();
- }
-
- @VisibleForTesting
- public long getErrorCount() {
- return errorCount.get();
- }
-
- @VisibleForTesting
- public long getPendingCrlRemoveCount() {
- return pendingCrlRemoveCount.get();
- }
-
- public ClientCRLStore getClientCRLStore() {
- return clientCRLStore;
- }
- public AtomicBoolean getIsRunning() {
- return isRunning;
- }
-
- public void stop(boolean shutdown) {
- LOG.info("{}: stopping...", CLIENT_NAME);
- if (isRunning.get()) {
- // complete update request, no more client streaming
- if (requestObserver != null) {
- requestObserver.onCompleted();
- requestObserver = null;
- }
-
- // stop update handler
- if (handler != null) {
- handler.stop();
- handler = null;
- }
-
- if (shutdown) {
- shutdownChannel();
- }
- isRunning.set(false);
- }
- LOG.info("{}: stopped.", CLIENT_NAME);
- }
-
- public void restart() {
- resetClient();
- stop(false);
- start();
- }
-
- public void createChannel() {
- NettyChannelBuilder channelBuilder =
- NettyChannelBuilder.forAddress(host, port).usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
-
- channel = channelBuilder.build();
- updateClient = SCMUpdateServiceGrpc.newStub(channel);
- subscribeClient = SCMUpdateServiceGrpc.newBlockingStub(channel);
- }
-
- public void shutdownChannel() {
- if (channel == null) {
- return;
- }
-
- channel.shutdown();
- try {
- channel.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- LOG.error("Failed to shutdown {} channel", CLIENT_NAME, e);
- Thread.currentThread().interrupt();
- } finally {
- channel.shutdownNow();
- channel = null;
- }
- }
-
- private UUID subScribeClient() {
- SubscribeRequest subReq = SubscribeRequest.newBuilder().build();
- SubscribeResponse subResp = subscribeClient.withWaitForReady()
- .subscribe(subReq);
- return SCMUpdateClientInfo.fromClientIdProto(subResp.getClientId());
- }
-
- private void unSubscribeClient() {
- if (clientId != null) {
- UnsubscribeRequest unsubReq = UnsubscribeRequest.newBuilder()
- .setClientId(SCMUpdateClientInfo.toClientIdProto(clientId)).build();
- subscribeClient.withWaitForReady().
- withDeadline(Deadline.after(5, TimeUnit.MILLISECONDS))
- .unsubscribe(unsubReq);
- }
- }
-
- // short-circuit the backoff timer and make them reconnect immediately.
- private void resetClient() {
- if (channel == null) {
- return;
- }
- channel.resetConnectBackoff();
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/UpdateServiceConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/UpdateServiceConfig.java
deleted file mode 100644
index 958335f5cf..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/UpdateServiceConfig.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.update.client;
-
-import org.apache.hadoop.hdds.conf.Config;
-import org.apache.hadoop.hdds.conf.ConfigGroup;
-import org.apache.hadoop.hdds.conf.ConfigTag;
-
-/**
- * Update service configuration.
- */
-@ConfigGroup(prefix = "ozone.scm.update.service")
-public final class UpdateServiceConfig {
-
- @Config(key = "port", defaultValue = "9893", tags = {ConfigTag.SECURITY},
- description = "Port used for the SCM grpc update service for CRL.")
- private int port;
-
- public int getPort() {
- return port;
- }
-
- public UpdateServiceConfig setPort(
- int portParam) {
- this.port = portParam;
- return this;
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/package-info.java
deleted file mode 100644
index 65183f56f1..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * CRL client package.
- */
-package org.apache.hadoop.hdds.scm.update.client;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/CRLClientInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/CRLClientInfo.java
deleted file mode 100644
index 33386db268..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/CRLClientInfo.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Class wrap CRL client info on server side.
- */
-public class CRLClientInfo {
- private final SCMUpdateClientInfo updateClientInfo;
- private long receivedCrlId;
- private List<Long> pendingCrlIds;
-
- public CRLClientInfo(SCMUpdateClientInfo clientInfo) {
- this.updateClientInfo = clientInfo;
- }
-
- public long getReceivedCrlId() {
- return receivedCrlId;
- }
-
- public void setReceivedCrlId(long receivedCrlId) {
- this.receivedCrlId = receivedCrlId;
- }
-
- public List<Long> getPendingCrlIds() {
- return Collections.unmodifiableList(pendingCrlIds);
- }
-
- public void setPendingCrlIds(List<Long> pendingCrlIds) {
- this.pendingCrlIds = new ArrayList<>(pendingCrlIds);
- }
-
- public SCMUpdateClientInfo getUpdateClientInfo() {
- return updateClientInfo;
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientInfo.java
deleted file mode 100644
index f03ae845e1..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientInfo.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-
-import java.util.UUID;
-
-/**
- * Wrapper class for scm update client on server side.
- */
-public class SCMUpdateClientInfo {
- private StreamObserver<UpdateResponse> responseObserver;
- private UUID clientId;
-
- public SCMUpdateClientInfo(UUID clientId) {
- this(clientId, null);
- }
-
- public SCMUpdateClientInfo(UUID clientId,
- StreamObserver<UpdateResponse> responseObserver) {
- this.clientId = clientId;
- this.responseObserver = responseObserver;
- }
-
- public UUID getClientId() {
- return clientId;
- }
-
- public static SCMUpdateServiceProtos.ClientId toClientIdProto(UUID uuid) {
- return SCMUpdateServiceProtos.ClientId.newBuilder()
- .setLsb(uuid.getLeastSignificantBits())
- .setMsb(uuid.getMostSignificantBits()).build();
- }
-
- public static UUID fromClientIdProto(
- SCMUpdateServiceProtos.ClientId clientId) {
- return new UUID(clientId.getMsb(), clientId.getLsb());
- }
-
- public StreamObserver<UpdateResponse> getResponseObserver() {
- return responseObserver;
- }
-
- public void setResponseObserver(
- StreamObserver<UpdateResponse> responseObserver) {
- this.responseObserver = responseObserver;
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java
deleted file mode 100644
index 07c58675e2..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * CRL server package.
- */
-package org.apache.hadoop.hdds.scm.update.server;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CRLApprover.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CRLApprover.java
deleted file mode 100644
index 8632743dd0..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CRLApprover.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.certificate.authority;
-
-import org.bouncycastle.cert.X509v2CRLBuilder;
-import org.bouncycastle.operator.OperatorCreationException;
-
-import java.security.cert.CRLException;
-import java.security.cert.X509CRL;
-
-/**
- * CRL Approver interface is used to sign CRLs.
- */
-public interface CRLApprover {
-
- /**
- * Signs a CRL.
- * @param builder - CRL builder instance with CRL info to be signed.
- * @return Signed CRL.
- * @throws CRLException - On Error
- * @throws OperatorCreationException - on Error.
- */
- X509CRL sign(X509v2CRLBuilder builder)
- throws CRLException, OperatorCreationException;
-
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index d74ee1ff0a..70bd5ad04e 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -23,19 +23,14 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import java.io.IOException;
-import java.math.BigInteger;
import java.security.cert.CertPath;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
-import java.util.Date;
import java.util.List;
-import java.util.Optional;
import java.util.concurrent.Future;
/**
@@ -106,29 +101,17 @@ public interface CertificateServer {
CertificateApprover.ApprovalType type, NodeType role,
String certSerialId) throws SCMSecurityException;
- /**
- * Revokes a Certificate issued by this CertificateServer.
- *
- * @param serialIDs - List of serial IDs of Certificates to be revoked.
- * @param reason - Reason for revocation.
- * @param revocationTime - Revocation time for the certificates.
- * @return Future that gives a list of certificates that were revoked.
- */
- Future<Optional<Long>> revokeCertificates(
- List<BigInteger> serialIDs,
- CRLReason reason,
- Date revocationTime);
-
/**
* List certificates.
- * @param role - role: OM/SCM/DN
- * @param startSerialId - start certificate serial id
- * @param count - max number of certificates returned in a batch
+ *
+ * @param role - role: OM/SCM/DN
+ * @param startSerialId - start certificate serial id
+ * @param count - max number of certificates returned in a batch
* @return List of X509 Certificates.
* @throws IOException - On Failure
*/
List<X509Certificate> listCertificate(NodeType role,
- long startSerialId, int count, boolean isRevoked) throws IOException;
+ long startSerialId, int count) throws IOException;
/**
* Reinitialise the certificate server withe the SCMMetastore during SCM
@@ -137,17 +120,4 @@ public interface CertificateServer {
*/
void reinitialize(SCMMetadataStore scmMetadataStore);
- /**
- * Get the CRLInfo based on the CRL Ids.
- * @param crlIds - list of crl ids
- * @return CRLInfo
- * @throws IOException
- */
- List<CRLInfo> getCrls(List<Long> crlIds) throws IOException;
-
- /**
- * Get the latest CRL id.
- * @return latest CRL id.
- */
- long getLatestCrlId();
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
index b5bfc54f33..2548f81095 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
@@ -20,21 +20,13 @@
package org.apache.hadoop.hdds.security.x509.certificate.authority;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.security.x509.crl.CRLStatus;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import java.io.IOException;
import java.math.BigInteger;
import java.security.cert.X509Certificate;
-import java.util.Date;
import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
/**
* This interface allows the DefaultCA to be portable and use different DB
@@ -74,40 +66,8 @@ public interface CertificateStore {
*/
void checkValidCertID(BigInteger serialID) throws IOException;
-
- /**
- * Adds the certificates to be revoked to a new CRL and moves all the
- * certificates in a transactional manner from valid certificate to
- * revoked certificate state. Returns an empty {@code Optional} instance if
- * the certificates were invalid / not found / already revoked and no CRL
- * was generated. Otherwise, returns the newly generated CRL sequence ID.
- * @param serialIDs - List of Serial IDs of Certificates to be revoked.
- * @param caCertificateHolder - X509 Certificate Holder of the CA.
- * @param reason - CRLReason for revocation.
- * @param revocationTime - Revocation Time for the certificates.
- * @param approver - CRL approver to sign the CRL.
- * @return An empty {@code Optional} instance if no CRL was generated.
- * Otherwise, returns the newly generated CRL sequence ID.
- * @throws IOException - on failure.
- */
- @Replicate
- Optional<Long> revokeCertificates(List<BigInteger> serialIDs,
- X509CertificateHolder caCertificateHolder,
- CRLReason reason,
- Date revocationTime,
- CRLApprover approver)
- throws IOException;
-
- /**
- * Deletes an expired certificate from the store. Please note: We don't
- * remove revoked certificates, we need that information to generate the
- * CRLs.
- * @param serialID - Certificate ID.
- */
- void removeExpiredCertificate(BigInteger serialID) throws IOException;
-
/**
- * Deletes all non-revoked expired certificates from the store.
+ * Deletes all expired certificates from the store.
*
* @return The list of removed expired certificates
* @throws IOException - on failure
@@ -117,37 +77,21 @@ public interface CertificateStore {
/**
* Retrieves a Certificate based on the Serial number of that certificate.
+ *
* @param serialID - ID of the certificate.
- * @param certType - Whether its Valid or Revoked certificate.
* @return X509Certificate
* @throws IOException - on failure.
*/
- X509Certificate getCertificateByID(BigInteger serialID, CertType certType)
- throws IOException;
+ X509Certificate getCertificateByID(BigInteger serialID) throws IOException;
/**
- * Retrieves a {@link CertInfo} for a revoked certificate based on the Serial
- * number of that certificate. This API can be used to get more information
- * like the timestamp when the certificate was persisted in the DB.
- * @param serialID - ID of the certificate.
- * @return CertInfo
- * @throws IOException - on failure.
- */
- CertInfo getRevokedCertificateInfoByID(BigInteger serialID)
- throws IOException;
-
- /**
- *
- * @param role - role of the certificate owner (OM/DN).
+ * @param role - role of the certificate owner (OM/DN).
* @param startSerialID - start cert serial id.
- * @param count - max number of certs returned.
- * @param certType cert type (valid/revoked).
+ * @param count - max number of certs returned.
* @return list of X509 certificates.
* @throws IOException - on failure.
*/
- List<X509Certificate> listCertificate(NodeType role,
- BigInteger startSerialID, int count, CertType certType)
- throws IOException;
+ List<X509Certificate> listCertificate(NodeType role, BigInteger startSerialID, int count) throws IOException;
/**
* Reinitialize the certificate server.
@@ -155,29 +99,4 @@ public interface CertificateStore {
*/
void reinitialize(SCMMetadataStore metadataStore);
- /**
- * Get the CRLInfo based on the CRL Ids.
- * @param crlIds - list of crl ids
- * @return CRLInfo
- * @throws IOException
- */
- List<CRLInfo> getCrls(List<Long> crlIds) throws IOException;
-
- /**
- * Get the latest CRL id.
- * @return latest CRL id.
- */
- long getLatestCrlId();
-
- CRLStatus getCRLStatusForDN(UUID uuid);
-
- void setCRLStatusForDN(UUID uuid, CRLStatus crlStatus);
-
- /**
- * Different kind of Certificate stores.
- */
- enum CertType {
- VALID_CERTS,
- REVOKED_CERTS
- }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
index 6200f2d8da..03de925d06 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hdds.security.x509.certificate.authority;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.security.SecurityConfig;
@@ -30,10 +29,8 @@ import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.certificate.utils.SelfSignedCertificate;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
-import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.operator.OperatorCreationException;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
@@ -58,7 +55,6 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.List;
-import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.concurrent.locks.Lock;
@@ -128,10 +124,8 @@ public class DefaultCAServer implements CertificateServer {
*/
private PKIProfile profile;
private CertificateApprover approver;
- private CRLApprover crlApprover;
private CertificateStore store;
private Lock lock;
- private static boolean testSecureFlag;
private BigInteger rootCertificateId;
/**
@@ -177,8 +171,6 @@ public class DefaultCAServer implements CertificateServer {
Consumer<SecurityConfig> caInitializer =
processVerificationStatus(status, type);
caInitializer.accept(securityConfig);
- crlApprover = new DefaultCRLApprover(securityConfig,
- getCAKeys().getPrivate());
}
@Override
@@ -212,8 +204,8 @@ public class DefaultCAServer implements CertificateServer {
@Override
public X509Certificate getCertificate(String certSerialId) throws
IOException {
- return store.getCertificateByID(new BigInteger(certSerialId),
- CertificateStore.CertType.VALID_CERTS);
+ return store.getCertificateByID(new BigInteger(certSerialId)
+ );
}
private KeyPair getCAKeys() throws IOException {
@@ -314,35 +306,10 @@ public class DefaultCAServer implements CertificateServer {
return xcert;
}
- @Override
- public Future<Optional<Long>> revokeCertificates(
- List<BigInteger> certificates,
- CRLReason reason,
- Date revocationTime) {
- CompletableFuture<Optional<Long>> revoked = new CompletableFuture<>();
- if (CollectionUtils.isEmpty(certificates)) {
- revoked.completeExceptionally(new SCMSecurityException(
- "Certificates cannot be null or empty"));
- return revoked;
- }
- try {
- revoked.complete(
- store.revokeCertificates(certificates,
- getCACertificate(), reason, revocationTime, crlApprover)
- );
- } catch (IOException ex) {
- LOG.error("Revoking the certificate failed.", ex.getCause());
- revoked.completeExceptionally(new SCMSecurityException(ex));
- }
- return revoked;
- }
-
@Override
public List<X509Certificate> listCertificate(NodeType role,
- long startSerialId, int count, boolean isRevoked) throws IOException {
- return store.listCertificate(role, BigInteger.valueOf(startSerialId), count,
- isRevoked ? CertificateStore.CertType.REVOKED_CERTS :
- CertificateStore.CertType.VALID_CERTS);
+ long startSerialId, int count) throws IOException {
+ return store.listCertificate(role, BigInteger.valueOf(startSerialId), count);
}
@Override
@@ -350,22 +317,6 @@ public class DefaultCAServer implements CertificateServer {
store.reinitialize(scmMetadataStore);
}
- /**
- * Get the CRLInfo based on the CRL Ids.
- * @param crlIds - list of crl ids
- * @return CRLInfo
- * @throws IOException
- */
- @Override
- public List<CRLInfo> getCrls(List<Long> crlIds) throws IOException {
- return store.getCrls(crlIds);
- }
-
- @Override
- public long getLatestCrlId() {
- return store.getLatestCrlId();
- }
-
/**
* Generates a Self Signed CertificateServer. These are the steps in
* generating a Self-Signed CertificateServer.
@@ -655,8 +606,4 @@ public class DefaultCAServer implements CertificateServer {
INITIALIZE /* All artifacts are missing, we should init the system. */
}
- @VisibleForTesting
- public static void setTestSecureFlag(boolean flag) {
- testSecureFlag = flag;
- }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCRLApprover.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCRLApprover.java
deleted file mode 100644
index 671e31d18d..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCRLApprover.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.certificate.authority;
-
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.crl.CRLCodec;
-import org.bouncycastle.cert.X509CRLHolder;
-import org.bouncycastle.cert.X509v2CRLBuilder;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-
-import java.security.PrivateKey;
-import java.security.cert.CRLException;
-import java.security.cert.X509CRL;
-
-/**
- * Default CRL Approver used by the DefaultCA.
- */
-public class DefaultCRLApprover implements CRLApprover {
-
- private SecurityConfig config;
- private PrivateKey caPrivate;
-
- public DefaultCRLApprover(SecurityConfig config, PrivateKey caPrivate) {
- this.config = config;
- this.caPrivate = caPrivate;
- }
-
- @Override
- public X509CRL sign(X509v2CRLBuilder builder)
- throws CRLException, OperatorCreationException {
- JcaContentSignerBuilder contentSignerBuilder =
- new JcaContentSignerBuilder(config.getSignatureAlgo());
-
- contentSignerBuilder.setProvider(config.getProvider());
- X509CRLHolder crlHolder =
- builder.build(contentSignerBuilder.build(caPrivate));
-
- return CRLCodec.getX509CRL(crlHolder);
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLCodec.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLCodec.java
deleted file mode 100644
index f634cd6dcc..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLCodec.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509.crl;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.bouncycastle.cert.X509CRLHolder;
-import org.bouncycastle.cert.jcajce.JcaX509CRLConverter;
-import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory;
-import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.cert.CRLException;
-import java.security.cert.X509CRL;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * CRL Codec Utility class used for reading and writing
- * X.509 CRL PEM encoded Streams.
- */
-public class CRLCodec {
- private static final Logger LOG =
- LoggerFactory.getLogger(CRLCodec.class);
- private static final JcaX509CRLConverter CRL_CONVERTER
- = new JcaX509CRLConverter();
- private final SecurityConfig securityConfig;
- private final Path location;
- private final Set<PosixFilePermission> permissionSet =
- Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
- .collect(Collectors.toSet());
-
- /**
- * The CRL Codec allows us to encode and decode.
- *
- * @param securityConfig
- */
- public CRLCodec(SecurityConfig securityConfig) {
- this.securityConfig = securityConfig;
- this.location = securityConfig.getCertificateLocation("scm");
- }
-
- /**
- * Returns a X509 CRL from the CRL Holder.
- *
- * @param holder - Holder
- * @return X509CRL - X509 CRL.
- * @throws CRLException - on Error.
- */
- public static X509CRL getX509CRL(X509CRLHolder holder)
- throws CRLException {
- return CRL_CONVERTER.getCRL(holder);
- }
-
- /**
- * Returns the Certificate as a PEM encoded String.
- *
- * @param holder - X.509 CRL Holder.
- * @return PEM Encoded Certificate String.
- * @throws SCMSecurityException - On failure to create a PEM String.
- */
- public static String getPEMEncodedString(X509CRLHolder holder)
- throws SCMSecurityException {
- LOG.trace("Getting PEM version of a CRL.");
- try {
- return getPEMEncodedString(getX509CRL(holder));
- } catch (CRLException exp) {
- throw new SCMSecurityException(exp);
- }
- }
-
- public static String getPEMEncodedString(X509CRL holder)
- throws SCMSecurityException {
- try {
- StringWriter stringWriter = new StringWriter();
- try (JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
- pemWriter.writeObject(holder);
- }
- return stringWriter.toString();
- } catch (IOException e) {
- throw new SCMSecurityException("PEM Encoding failed for CRL." +
- holder.getIssuerDN().toString(), e);
- }
- }
-
- /**
- * Gets the X.509 CRL from PEM encoded String.
- *
- * @param pemEncodedString - PEM encoded String.
- * @return X509CRL - Crl.
- * @throws CRLException - Thrown on Failure.
- */
- public static X509CRL getX509CRL(String pemEncodedString)
- throws CRLException {
- return getX509CRL(pemEncodedString, Function.identity());
- }
-
- public static <E extends Exception> X509CRL getX509CRL(String pemEncoded,
- Function<CRLException, E> convertor)
- throws E {
- CertificateFactory fact = CertificateCodec.getCertFactory();
- // ByteArrayInputStream.close(), which is a noop, can be safely ignored.
- final ByteArrayInputStream input = new ByteArrayInputStream(
- pemEncoded.getBytes(UTF_8));
- try {
- return (X509CRL) fact.engineGenerateCRL(input);
- } catch (CRLException e) {
- throw convertor.apply(e);
- }
- }
-
- public static IOException toIOException(CRLException e) {
- return new IOException("Failed to engineGenerateCRL", e);
- }
-
- /**
- * Get CRL location.
- *
- * @return Path
- */
- public Path getLocation() {
- return location;
- }
-
- /**
- * Write the CRL pointed to the location by the configs.
- *
- * @param crl - X509CRL CRL to write.
- * @throws IOException - on Error.
- */
- public void writeCRL(X509CRL crl)
- throws IOException {
- String pem = getPEMEncodedString(crl);
- writeCRL(location.toAbsolutePath(),
- this.securityConfig.getCrlName(), pem, false);
- }
-
- /**
- * Write the CRL to the specific file.
- *
- * @param crlHolder - CRL to write.
- * @param fileName - file name to write to.
- * @param overwrite - boolean value, true means overwrite an existing
- * crl.
- * @throws IOException - On Error.
- */
- public void writeCRL(X509CRLHolder crlHolder,
- String fileName, boolean overwrite)
- throws IOException {
- String pem = getPEMEncodedString(crlHolder);
- writeCRL(location.toAbsolutePath(), fileName, pem, overwrite);
- }
-
- /**
- * Write the CRL to the specific file.
- *
- * @param basePath - Base Path where CRL file to be written.
- * @param fileName - file name of CRL file.
- * @param pemCRLString - PEN Encoded string
- * @param force - boolean value, true means overwrite an existing
- * crl.
- * @throws IOException - On Error.
- */
- public synchronized void writeCRL(Path basePath, String fileName,
- String pemCRLString, boolean force)
- throws IOException {
- File crlFile =
- Paths.get(basePath.toString(), fileName).toFile();
- if (crlFile.exists() && !force) {
- throw new SCMSecurityException("Specified CRL file already " +
- "exists.Please use force option if you want to overwrite it.");
- }
- // if file exists otherwise, if able to create
- if (!basePath.toFile().exists() && !basePath.toFile().mkdirs()) {
- LOG.error("Unable to create file path. Path: {}", basePath);
- throw new IOException("Creation of the directories failed."
- + basePath.toString());
- }
- try (FileOutputStream file = new FileOutputStream(crlFile)) {
- IOUtils.write(pemCRLString, file, UTF_8);
- }
- Files.setPosixFilePermissions(crlFile.toPath(), permissionSet);
- }
-
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java
deleted file mode 100644
index 134c841e69..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.security.x509.crl;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CRLInfoProto;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.utils.db.Codec;
-import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
-import org.apache.hadoop.hdds.utils.db.Proto2Codec;
-import jakarta.annotation.Nonnull;
-
-import java.security.cert.CRLException;
-import java.security.cert.X509CRL;
-import java.security.cert.X509CRLEntry;
-import java.time.Instant;
-import java.util.Comparator;
-import java.util.Objects;
-import java.util.function.Function;
-
-/**
- * Class that wraps Certificate Revocation List Info.
- */
-public final class CRLInfo implements Comparator<CRLInfo>,
- Comparable<CRLInfo> {
-
- private static final Codec<CRLInfo> CODEC = new DelegatedCodec<>(
- Proto2Codec.get(CRLInfoProto.getDefaultInstance()),
- proto -> fromProtobuf(proto, CRLCodec::toIOException),
- CRLInfo::getProtobuf);
-
- public static Codec<CRLInfo> getCodec() {
- return CODEC;
- }
-
- private final X509CRL x509CRL;
- private final long creationTimestamp;
- private final long crlSequenceID;
- private final Instant revocationTime;
-
- private CRLInfo(X509CRL x509CRL, long creationTimestamp, long crlSequenceID) {
- assert ((x509CRL != null) &&
- !x509CRL.getRevokedCertificates().isEmpty());
- this.x509CRL = x509CRL;
- this.creationTimestamp = creationTimestamp;
- this.crlSequenceID = crlSequenceID;
- X509CRLEntry entry = x509CRL.getRevokedCertificates().iterator().next();
- this.revocationTime = Instant.ofEpochMilli(
- entry.getRevocationDate().getTime());
- }
-
- public static CRLInfo fromProtobuf(CRLInfoProto info)
- throws CRLException {
- return fromProtobuf(info, Function.identity());
- }
-
- private static <E extends Exception> CRLInfo fromProtobuf(
- CRLInfoProto info, Function<CRLException, E> convertor) throws E {
- return new CRLInfo.Builder()
- .setX509CRL(CRLCodec.getX509CRL(info.getX509CRL(), convertor))
- .setCreationTimestamp(info.getCreationTimestamp())
- .setCrlSequenceID(info.getCrlSequenceID())
- .build();
- }
-
- public CRLInfoProto getProtobuf() throws SCMSecurityException {
- return CRLInfoProto.newBuilder()
- .setX509CRL(CRLCodec.getPEMEncodedString(getX509CRL()))
- .setCreationTimestamp(getCreationTimestamp())
- .setCrlSequenceID(getCrlSequenceID())
- .build();
- }
-
- public static CRLInfo fromCRLProto3(
- SCMUpdateServiceProtos.CRLInfoProto info)
- throws CRLException {
- return new CRLInfo.Builder()
- .setX509CRL(CRLCodec.getX509CRL(info.getX509CRL()))
- .setCreationTimestamp(info.getCreationTimestamp())
- .setCrlSequenceID(info.getCrlSequenceID())
- .build();
- }
-
- public SCMUpdateServiceProtos.CRLInfoProto getCRLProto3()
- throws SCMSecurityException {
- return SCMUpdateServiceProtos.CRLInfoProto.newBuilder()
- .setX509CRL(CRLCodec.getPEMEncodedString(getX509CRL()))
- .setCreationTimestamp(getCreationTimestamp())
- .setCrlSequenceID(getCrlSequenceID())
- .build();
- }
-
- public X509CRL getX509CRL() {
- return x509CRL;
- }
-
- public long getCreationTimestamp() {
- return creationTimestamp;
- }
-
- public long getCrlSequenceID() {
- return crlSequenceID;
- }
-
- public boolean shouldRevokeNow() {
- return revocationTime.isBefore(Instant.now());
- }
-
- public Instant getRevocationTime() {
- return revocationTime;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object
- * is less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(@Nonnull CRLInfo o) {
- return this.compare(this, o);
- }
-
- /**
- * Compares its two arguments for order. Returns a negative integer,
- * zero, or a positive integer as the first argument is less than, equal
- * to, or greater than the second.<p>
- * <p>
- *
- * @param o1 the first object to be compared.
- * @param o2 the second object to be compared.
- * @return a negative integer, zero, or a positive integer as the
- * first argument is less than, equal to, or greater than the
- * second.
- * @throws NullPointerException if an argument is null and this
- * comparator does not permit null arguments
- * @throws ClassCastException if the arguments' types prevent them from
- * being compared by this comparator.
- */
- @Override
- public int compare(CRLInfo o1, CRLInfo o2) {
- return Long.compare(o1.getCreationTimestamp(), o2.getCreationTimestamp());
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- CRLInfo that = (CRLInfo) o;
-
- return this.crlSequenceID == that.crlSequenceID &&
- this.getX509CRL().equals(that.x509CRL) &&
- this.creationTimestamp == that.creationTimestamp;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getX509CRL(), getCreationTimestamp());
- }
-
- @Override
- public String toString() {
- return "CRLInfo{" +
- "crlSequenceID=" + crlSequenceID +
- ", x509CRL=" + x509CRL.toString() +
- ", creationTimestamp=" + creationTimestamp +
- '}';
- }
-
- /**
- * Builder class for CRLInfo.
- */
- @SuppressWarnings("checkstyle:hiddenfield")
- public static class Builder {
- private X509CRL x509CRL;
- private long creationTimestamp;
- private long crlSequenceID;
-
- public Builder setX509CRL(X509CRL x509CRL) {
- this.x509CRL = x509CRL;
- return this;
- }
-
- public Builder setCreationTimestamp(long creationTimestamp) {
- this.creationTimestamp = creationTimestamp;
- return this;
- }
-
- public Builder setCrlSequenceID(long crlSequenceID) {
- this.crlSequenceID = crlSequenceID;
- return this;
- }
-
- public CRLInfo build() {
- return new CRLInfo(x509CRL, creationTimestamp, crlSequenceID);
- }
- }
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java
deleted file mode 100644
index 9f768a3263..0000000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains the common routines used for maintaining Certificate
- * Revocation Lists.
- */
-package org.apache.hadoop.hdds.security.x509.crl;
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
index 9b3125c11b..ea07a79303 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
@@ -19,11 +19,6 @@
package org.apache.hadoop.hdds.security.x509.certificate.authority;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.security.x509.crl.CRLStatus;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import java.io.IOException;
@@ -31,10 +26,7 @@ import java.math.BigInteger;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
/**
*
@@ -56,67 +48,25 @@ public class MockCAStore implements CertificateStore {
X509Certificate certificate) throws IOException {
}
- @Override
- public Optional<Long> revokeCertificates(
- List<BigInteger> serialIDs,
- X509CertificateHolder caCertificateHolder,
- CRLReason reason,
- Date revocationTime,
- CRLApprover approver) throws IOException {
- return Optional.empty();
- }
-
- @Override
- public void removeExpiredCertificate(BigInteger serialID)
- throws IOException {
-
- }
-
@Override
public List<X509Certificate> removeAllExpiredCertificates() {
return new ArrayList<>();
}
@Override
- public X509Certificate getCertificateByID(BigInteger serialID,
- CertType certType)
- throws IOException {
- return null;
- }
-
- @Override
- public CertInfo getRevokedCertificateInfoByID(BigInteger serialID)
+ public X509Certificate getCertificateByID(BigInteger serialID)
throws IOException {
return null;
}
@Override
public List<X509Certificate> listCertificate(NodeType role,
- BigInteger startSerialID, int count, CertType certType)
+ BigInteger startSerialID, int count)
throws IOException {
return Collections.emptyList();
}
@Override
- public void reinitialize(SCMMetadataStore metadataStore) { }
-
- @Override
- public List<CRLInfo> getCrls(List<Long> crlIds) throws IOException {
- return Collections.emptyList();
- }
-
- @Override
- public long getLatestCrlId() {
- return 0;
- }
-
- @Override
- public CRLStatus getCRLStatusForDN(UUID uuid) {
- return null;
- }
-
- @Override
- public void setCRLStatusForDN(UUID uuid, CRLStatus crlStatus) {
-
+ public void reinitialize(SCMMetadataStore metadataStore) {
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
index f98ad41e8d..740c3258a5 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
@@ -45,7 +44,6 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
-import java.math.BigInteger;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyPair;
@@ -58,11 +56,7 @@ import java.security.cert.X509Certificate;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
import java.util.List;
-import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -243,55 +237,6 @@ public class TestDefaultCAServer {
assertNotNull(CertificateCodec.firstCertificateFrom(holder.get()));
}
- @Test
- public void testRevokeCertificates() throws Exception {
- String scmId = RandomStringUtils.randomAlphabetic(4);
- String clusterId = RandomStringUtils.randomAlphabetic(4);
- Date now = new Date();
-
- CertificateServer testCA = new DefaultCAServer("testCA",
- clusterId, scmId, caStore,
- new DefaultProfile(),
- Paths.get(SCM_CA_CERT_STORAGE_DIR, SCM_CA_PATH).toString());
- testCA.init(securityConfig, CAType.ROOT);
-
- KeyPair keyPair =
- new HDDSKeyGenerator(securityConfig).generateKey();
- PKCS10CertificationRequest csr = new CertificateSignRequest.Builder()
- .addDnsName("hadoop.apache.org")
- .addIpAddress("8.8.8.8")
- .setCA(false)
- .setSubject("testCA")
- .setConfiguration(securityConfig)
- .setKey(keyPair)
- .build();
-
- Future<CertPath> holder = testCA.requestCertificate(
- csr, CertificateApprover.ApprovalType.TESTING_AUTOMATIC, OM,
- String.valueOf(System.nanoTime()));
-
- X509Certificate certificate =
- CertificateCodec.firstCertificateFrom(holder.get());
- List<BigInteger> serialIDs = new ArrayList<>();
- serialIDs.add(certificate.getSerialNumber());
- Future<Optional<Long>> revoked = testCA.revokeCertificates(serialIDs,
- CRLReason.lookup(CRLReason.keyCompromise), now);
-
- // Revoking a valid certificate complete successfully without errors.
- assertTrue(revoked.isDone());
-
- // Revoking empty list of certificates should throw an error.
- ExecutionException execution = assertThrows(ExecutionException.class,
- () -> {
- Future<Optional<Long>> result =
- testCA.revokeCertificates(Collections.emptyList(),
- CRLReason.lookup(CRLReason.keyCompromise), now);
- result.get();
- });
- assertThat(execution.getCause().getMessage())
- .contains("Certificates cannot be null");
- }
-
@Test
public void testRequestCertificateWithInvalidSubjectFailure()
throws Exception {
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCRLCodec.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCRLCodec.java
deleted file mode 100644
index d50dbbecfd..0000000000
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestCRLCodec.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.certificate.utils;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.cert.CRLException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509CRL;
-import java.time.LocalDateTime;
-import java.util.Date;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.crl.CRLCodec;
-import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CRLEntryHolder;
-import org.bouncycastle.cert.X509CRLHolder;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v2CRLBuilder;
-import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory;
-import org.bouncycastle.operator.OperatorCreationException;
-import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-/**
- * Tests for the CRLCodec.
- */
-public class TestCRLCodec {
-
- private static final String COMPONENT = "test";
- private SecurityConfig securityConfig;
- private X509CertificateHolder x509CertificateHolder;
- private KeyPair keyPair;
- private static final String TMP_CERT_FILE_NAME = "pemcertificate.crt";
-
- private File basePath;
- private static final String TMP_CRL_ENTRY =
- "-----BEGIN X509 CRL-----\n" +
- "MIIBijB0AgEBMA0GCSqGSIb3DQEBCwUAMC0xDTALBgNVBAMMBHdxVG0xDTALBgNV\n" +
- "BAsMBGVFY2gxDTALBgNVBAoMBHJpc1UXDTIwMDExNzE4NTcyMFowIjAgAgEBFw0y\n" +
- "MDAxMTcxODU3MjBaMAwwCgYDVR0VBAMKAQIwDQYJKoZIhvcNAQELBQADggEBACRI\n" +
- "i/nFK2/5rsNWAsYjT/Byhq6shQy+EjdvSzs2cezHbO2TKXnIhlHbvTp5JO/ClaGm\n" +
- "yfdwH6OjQbujcjceSKGSDQwNm98/JsryUh17IWcKJa9dlqFSUCy7GTZaXK6a3nH8\n" +
- "SNhcqzrR69lLc4vJZAy0FkmBCnjbdUX8I92ZHfNQNJaC4JQ8JFtjfzZCcQR9KZxw\n" +
- "bVue37JByiTxmxoiiMZf3MpOccuWKsZzIr9Tiw9G9inPS8lxRXODruDtMTpR8NPB\n" +
- "KL0Yg+JEV48v2GJ5kSObuawCD2uDDNpHDd6q2m1z6J69z5IYpWb8OHEyQT7J4u+b\n" +
- "tPiRCAUQLW9BACm17xc=\n" +
- "-----END X509 CRL-----\n";
-
- @BeforeEach
- public void init(@TempDir Path tempDir) throws NoSuchProviderException,
- NoSuchAlgorithmException, IOException,
- CertificateException, OperatorCreationException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OZONE_METADATA_DIRS, tempDir.toString());
- securityConfig = new SecurityConfig(conf);
- writeTempCert();
- x509CertificateHolder = readTempCert();
- }
-
-
- @Test
- public void testWriteCRL() throws IOException, OperatorCreationException {
-
- X500Name issuer = x509CertificateHolder.getIssuer();
- Date now = new Date();
- X509v2CRLBuilder builder = new X509v2CRLBuilder(issuer, now);
- builder.addCRLEntry(x509CertificateHolder.getSerialNumber(), now,
- CRLReason.cACompromise);
-
- JcaContentSignerBuilder contentSignerBuilder =
- new JcaContentSignerBuilder(securityConfig.getSignatureAlgo());
-
- contentSignerBuilder.setProvider(securityConfig.getProvider());
- PrivateKey privateKey = keyPair.getPrivate();
- X509CRLHolder cRLHolder =
- builder.build(contentSignerBuilder.build(privateKey));
-
- CRLCodec crlCodec = new CRLCodec(securityConfig);
- crlCodec.writeCRL(cRLHolder, this.securityConfig.getCrlName(), true);
-
- X509CRLEntryHolder entryHolder =
- cRLHolder.getRevokedCertificate(BigInteger.ONE);
- assertNotNull(entryHolder);
-
- // verify file generation
- File crlFile =
- Paths.get(crlCodec.getLocation().toString(),
- this.securityConfig.getCrlName()).toFile();
- assertTrue(crlFile.exists());
-
- try (BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream(crlFile), UTF_8))) {
-
- // Verify contents of the file
- String header = reader.readLine();
- assertEquals("-----BEGIN X509 CRL-----", header);
-
- String footer = null;
- String line = null;
- while ((line = reader.readLine()) != null) {
- footer = line;
- }
- assertEquals("-----END X509 CRL-----", footer);
- }
- }
-
- @Test
- public void testWriteCRLX509() throws IOException,
- OperatorCreationException, CertificateException, CRLException {
-
- X500Name issuer = x509CertificateHolder.getIssuer();
- Date now = new Date();
- X509v2CRLBuilder builder = new X509v2CRLBuilder(issuer, now);
- builder.addCRLEntry(x509CertificateHolder.getSerialNumber(), now,
- CRLReason.cACompromise);
-
- byte[] crlBytes = TMP_CRL_ENTRY.getBytes(UTF_8);
- try (InputStream inStream = new ByteArrayInputStream(crlBytes)) {
- CertificateFactory cf = CertificateCodec.getCertFactory();
- X509CRL crl = (X509CRL) cf.engineGenerateCRL(inStream);
-
- CRLCodec crlCodec = new CRLCodec(securityConfig);
- crlCodec.writeCRL(crl);
-
- // verify file generated or not
- File crlFile =
- Paths.get(crlCodec.getLocation().toString(),
- this.securityConfig.getCrlName()).toFile();
-
- assertTrue(crlFile.exists());
- }
- }
-
- @Test
- public void testGetX509CRL() throws IOException,
- OperatorCreationException, CertificateException, CRLException {
-
- X500Name issuer = x509CertificateHolder.getIssuer();
- Date now = new Date();
- X509v2CRLBuilder builder = new X509v2CRLBuilder(issuer, now);
- builder.addCRLEntry(x509CertificateHolder.getSerialNumber(), now,
- CRLReason.cACompromise);
-
- JcaContentSignerBuilder contentSignerBuilder =
- new JcaContentSignerBuilder(securityConfig.getSignatureAlgo());
-
- contentSignerBuilder.setProvider(securityConfig.getProvider());
- PrivateKey privateKey = keyPair.getPrivate();
- X509CRLHolder cRLHolder =
- builder.build(contentSignerBuilder.build(privateKey));
-
- CRLCodec crlCodec = new CRLCodec(securityConfig);
- crlCodec.writeCRL(cRLHolder, this.securityConfig.getCrlName(), true);
-
- X509CRLEntryHolder entryHolder =
- cRLHolder.getRevokedCertificate(BigInteger.ONE);
- assertNotNull(entryHolder);
-
- String pemEncodedString = crlCodec.getPEMEncodedString(cRLHolder);
- assertNotNull(pemEncodedString);
-
- // Verify header and footer of PEM encoded String
- String header = "-----BEGIN X509 CRL-----";
- String footer = "-----END X509 CRL-----";
- assertThat(pemEncodedString).contains(header);
- assertThat(pemEncodedString).contains(footer);
- }
-
- @Test
- public void testGetX509CRLFromCRLHolder() throws IOException,
- OperatorCreationException, CertificateException, CRLException {
-
- X500Name issuer = x509CertificateHolder.getIssuer();
- Date now = new Date();
- X509v2CRLBuilder builder = new X509v2CRLBuilder(issuer, now);
- builder.addCRLEntry(x509CertificateHolder.getSerialNumber(), now,
- CRLReason.cACompromise);
-
- JcaContentSignerBuilder contentSignerBuilder =
- new JcaContentSignerBuilder(securityConfig.getSignatureAlgo());
-
- contentSignerBuilder.setProvider(securityConfig.getProvider());
- PrivateKey privateKey = keyPair.getPrivate();
- X509CRLHolder cRLHolder =
- builder.build(contentSignerBuilder.build(privateKey));
-
- CRLCodec crlCodec = new CRLCodec(securityConfig);
-
- X509CRL crl = crlCodec.getX509CRL(cRLHolder);
- assertNotNull(crl);
- }
-
- /**
- * Test method for generating temporary cert and persisting into tmp folder.
- *
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- * @throws IOException
- */
- private void writeTempCert() throws NoSuchProviderException,
- NoSuchAlgorithmException, IOException {
- HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig);
- keyPair = keyGenerator.generateKey();
- LocalDateTime startDate = LocalDateTime.now();
- LocalDateTime endDate = startDate.plusDays(1);
- X509CertificateHolder cert =
- SelfSignedCertificate.newBuilder()
- .setSubject(RandomStringUtils.randomAlphabetic(4))
- .setClusterID(RandomStringUtils.randomAlphabetic(4))
- .setScmID(RandomStringUtils.randomAlphabetic(4))
- .setBeginDate(startDate)
- .setEndDate(endDate)
- .setConfiguration(securityConfig)
- .setKey(keyPair)
- .makeCA()
- .build();
- CertificateCodec codec =
- new CertificateCodec(securityConfig, COMPONENT);
-
- String pemString = codec.getPEMEncodedString(cert);
- basePath = new File(
- String.valueOf(
- securityConfig.getCertificateLocation("scm")));
-
- if (!basePath.exists()) {
- assertTrue(basePath.mkdirs());
- }
- codec.writeCertificate(basePath.toPath(), TMP_CERT_FILE_NAME,
- pemString);
- }
-
- private X509CertificateHolder readTempCert()
- throws IOException, CertificateException {
-
- CertificateCodec codec =
- new CertificateCodec(securityConfig, COMPONENT);
-
- X509CertificateHolder x509CertHolder =
- codec.getTargetCertHolder(basePath.toPath(), TMP_CERT_FILE_NAME);
-
- assertNotNull(x509CertHolder);
-
- return x509CertHolder;
- }
-}
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
index 3d281975f2..7778a6828f 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
@@ -125,12 +125,15 @@ enum Status {
FAILED_TO_INIT_PIPELINE_CHOOSE_POLICY = 30;
FAILED_TO_INIT_LEADER_CHOOSE_POLICY = 31;
SCM_NOT_LEADER = 32;
+ // not used anymore
FAILED_TO_REVOKE_CERTIFICATES = 33;
PIPELINE_NOT_FOUND = 34;
UNKNOWN_PIPELINE_STATE = 35;
CONTAINER_NOT_FOUND = 36;
CONTAINER_REPLICA_NOT_FOUND = 37;
+ // not used anymore
FAILED_TO_CONNECT_TO_CRL_SERVICE = 38;
+ //not used anymore
FAILED_TO_ADD_CRL_CLIENT = 39;
INVALID_PIPELINE_STATE = 40;
DUPLICATED_PIPELINE_ID = 41;
diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
index adc2eee385..f4fcabdc19 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto
@@ -52,6 +52,8 @@ message SCMSecurityRequest {
optional SCMListCertificateRequestProto listCertificateRequest = 7;
optional SCMGetSCMCertRequestProto getSCMCertificateRequest = 8;
optional SCMListCACertificateRequestProto listCACertificateRequestProto = 9;
+ //10, 11 and 12 are removed from the code, kept here to preserve the numbers.
+ //once we switch to protoc3 for compile, we can remove these and reserve the numbers and names.
optional SCMGetCrlsRequestProto getCrlsRequest = 10;
optional SCMGetLatestCrlIdRequestProto getLatestCrlIdRequest = 11;
optional SCMRevokeCertificatesRequestProto revokeCertificatesRequest = 12;
@@ -79,6 +81,8 @@ message SCMSecurityResponse {
optional SCMListCertificateResponseProto listCertificateResponseProto = 7;
+ //8, 9 and 10 are removed from the code, kept here to preserve the numbers.
+ //once we switch to protoc3 for compile, we can remove these and reserve the numbers and names.
optional SCMGetCrlsResponseProto getCrlsResponseProto = 8;
optional SCMGetLatestCrlIdResponseProto getLatestCrlIdResponseProto = 9;
@@ -99,6 +103,8 @@ enum Type {
GetSCMCertificate = 6;
GetRootCACertificate = 7;
ListCACertificate = 8;
+ //9, 10 and 11 are removed from the code, kept here to preserve the numbers.
+ //once we switch to protoc3 for compile, we can remove these and reserve the numbers and names.
GetCrls = 9;
GetLatestCrlId = 10;
RevokeCertificates = 11;
@@ -124,6 +130,8 @@ enum Status {
BLOCK_TOKEN_VERIFICATION_FAILED = 14;
GET_ROOT_CA_CERTIFICATE_FAILED = 15;
NOT_A_PRIMARY_SCM = 16;
+ //17 are removed from the code, kept here to preserve the numbers.
+ //once we switch to protoc3 for compile, we can remove these and reserve the numbers and names.
REVOKE_CERTIFICATE_FAILED = 17;
}
/**
@@ -223,10 +231,14 @@ message SCMListCACertificateRequestProto {
/**
* Proto request to get CRL.
*/
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMGetCrlsRequestProto {
repeated int64 crlId = 1;
}
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMGetCrlsResponseProto {
repeated CRLInfoProto crlInfos = 1;
}
@@ -234,13 +246,19 @@ message SCMGetCrlsResponseProto {
/**
* Proto request to get latest CRL id.
*/
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMGetLatestCrlIdRequestProto {
}
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMGetLatestCrlIdResponseProto {
optional int64 crlId = 1;
}
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMRevokeCertificatesRequestProto {
// Match with org.bouncycastle.asn1.x509.CRLReason
enum Reason {
@@ -263,6 +281,8 @@ message SCMRevokeCertificatesRequestProto {
message SCMGetAllRootCaCertificatesRequestProto {
}
+// Not used in the code anymore, should be removed later on, once we updated to protoc3 and properly reserve
+// places where we used this earlier.
message SCMRevokeCertificatesResponseProto {
optional int64 crlId = 1;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 0cc205b2ff..4fcf130117 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatusReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
@@ -208,15 +207,6 @@ public final class SCMEvents {
new TypedEvent<>(CommandStatusReportHandler.DeleteBlockStatus.class,
"Delete_Block_Status");
- /**
- * A CRL status report will be sent by datanodes. This report is received
- * and processed by SCMDatanodeHeartbeatDispatcher.
- */
- public static final TypedEvent<CRLStatusReportFromDatanode>
- CRL_STATUS_REPORT =
- new TypedEvent<>(CRLStatusReportFromDatanode.class,
- "Crl_Status_Report");
-
/**
* Private Ctor. Never Constructed.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
index 93271d1474..4426d18d8d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.metadata.Replicate;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import org.apache.hadoop.hdds.utils.UniqueId;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
@@ -430,16 +429,6 @@ public class SequenceIdGenerator {
}
}
- try (TableIterator<BigInteger,
- ? extends KeyValue<BigInteger, CertInfo>> iterator =
- scmMetadataStore.getRevokedCertsV2Table().iterator()) {
- while (iterator.hasNext()) {
- X509Certificate cert =
- iterator.next().getValue().getX509Certificate();
- largestCertId = Long.max(
- cert.getSerialNumber().longValueExact(), largestCertId);
- }
- }
sequenceIdTable.put(CERTIFICATE_ID, largestCertId);
LOG.info("upgrade {} to {}", CERTIFICATE_ID,
sequenceIdTable.get(CERTIFICATE_ID));
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
index 8b9b76df89..4a280d2103 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
@@ -27,9 +27,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import org.apache.hadoop.hdds.utils.TransactionInfo;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.utils.db.ByteStringCodec;
@@ -78,28 +76,6 @@ public class SCMDBDefinition extends DBDefinition.WithMap {
X509Certificate.class,
X509CertificateCodec.get());
- /**
- * This column family is Deprecated in favor of REVOKED_CERTS_V2.
- */
- @Deprecated
- public static final DBColumnFamilyDefinition<BigInteger, X509Certificate>
- REVOKED_CERTS =
- new DBColumnFamilyDefinition<>(
- "revokedCerts",
- BigInteger.class,
- BigIntegerCodec.get(),
- X509Certificate.class,
- X509CertificateCodec.get());
-
- public static final DBColumnFamilyDefinition<BigInteger, CertInfo>
- REVOKED_CERTS_V2 =
- new DBColumnFamilyDefinition<>(
- "revokedCertsV2",
- BigInteger.class,
- BigIntegerCodec.get(),
- CertInfo.class,
- CertInfo.getCodec());
-
public static final DBColumnFamilyDefinition<PipelineID, Pipeline>
PIPELINES =
new DBColumnFamilyDefinition<>(
@@ -127,23 +103,6 @@ public class SCMDBDefinition extends DBDefinition.WithMap {
TransactionInfo.class,
TransactionInfo.getCodec());
- public static final DBColumnFamilyDefinition<Long, CRLInfo> CRLS =
- new DBColumnFamilyDefinition<>(
- "crls",
- Long.class,
- LongCodec.get(),
- CRLInfo.class,
- CRLInfo.getCodec());
-
- public static final DBColumnFamilyDefinition<String, Long>
- CRL_SEQUENCE_ID =
- new DBColumnFamilyDefinition<>(
- "crlSequenceId",
- String.class,
- StringCodec.get(),
- Long.class,
- LongCodec.get());
-
public static final DBColumnFamilyDefinition<String, Long>
SEQUENCE_ID =
new DBColumnFamilyDefinition<>(
@@ -187,14 +146,10 @@ public class SCMDBDefinition extends DBDefinition.WithMap {
private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
CONTAINERS,
- CRLS,
- CRL_SEQUENCE_ID,
DELETED_BLOCKS,
META,
MOVE,
PIPELINES,
- REVOKED_CERTS,
- REVOKED_CERTS_V2,
SEQUENCE_ID,
STATEFUL_SERVICE_CONFIG,
TRANSACTIONINFO,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java
index cbff9527bb..ec63076b4a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java
@@ -30,27 +30,19 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS;
-import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CRLS;
-import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CRL_SEQUENCE_ID;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.DELETED_BLOCKS;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.MOVE;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES;
-import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.REVOKED_CERTS;
-import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.REVOKED_CERTS_V2;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.STATEFUL_SERVICE_CONFIG;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.TRANSACTIONINFO;
import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_CERTS;
@@ -75,20 +67,12 @@ public class SCMMetadataStoreImpl implements SCMMetadataStore {
private Table<BigInteger, X509Certificate> validSCMCertsTable;
- private Table<BigInteger, X509Certificate> revokedCertsTable;
-
- private Table<BigInteger, CertInfo> revokedCertsV2Table;
-
private Table<ContainerID, ContainerInfo> containerTable;
private Table<PipelineID, Pipeline> pipelineTable;
private Table<String, TransactionInfo> transactionInfoTable;
- private Table<Long, CRLInfo> crlInfoTable;
-
- private Table<String, Long> crlSequenceIdTable;
-
private Table<String, Long> sequenceIdTable;
private Table<ContainerID, MoveDataNodePair> moveTable;
@@ -152,14 +136,6 @@ public class SCMMetadataStoreImpl implements SCMMetadataStore {
checkAndPopulateTable(validSCMCertsTable, VALID_SCM_CERTS.getName());
- revokedCertsTable = REVOKED_CERTS.getTable(store);
-
- checkAndPopulateTable(revokedCertsTable, REVOKED_CERTS.getName());
-
- revokedCertsV2Table = REVOKED_CERTS_V2.getTable(store);
-
- checkAndPopulateTable(revokedCertsV2Table, REVOKED_CERTS_V2.getName());
-
pipelineTable = PIPELINES.getTable(store);
checkAndPopulateTable(pipelineTable, PIPELINES.getName());
@@ -172,14 +148,6 @@ public class SCMMetadataStoreImpl implements SCMMetadataStore {
checkAndPopulateTable(transactionInfoTable, TRANSACTIONINFO.getName());
- crlInfoTable = CRLS.getTable(store);
-
- checkAndPopulateTable(crlInfoTable, CRLS.getName());
-
- crlSequenceIdTable = CRL_SEQUENCE_ID.getTable(store);
-
- checkAndPopulateTable(crlInfoTable, CRL_SEQUENCE_ID.getName());
-
sequenceIdTable = SEQUENCE_ID.getTable(store);
checkAndPopulateTable(sequenceIdTable, SEQUENCE_ID.getName());
@@ -227,51 +195,6 @@ public class SCMMetadataStoreImpl implements SCMMetadataStore {
return validSCMCertsTable;
}
- @Override
- public Table<BigInteger, X509Certificate> getRevokedCertsTable() {
- return revokedCertsTable;
- }
-
- @Override
- public Table<BigInteger, CertInfo> getRevokedCertsV2Table() {
- return revokedCertsV2Table;
- }
-
- /**
- * A table that maintains X509 Certificate Revocation Lists and its metadata.
- *
- * @return Table.
- */
- @Override
- public Table<Long, CRLInfo> getCRLInfoTable() {
- return crlInfoTable;
- }
-
- /**
- * A table that maintains the last CRL SequenceId. This helps to make sure
- * that the CRL Sequence Ids are monotonically increasing.
- *
- * @return Table.
- */
- @Override
- public Table<String, Long> getCRLSequenceIdTable() {
- return crlSequenceIdTable;
- }
-
- @Override
- public TableIterator getAllCerts(CertificateStore.CertType certType)
- throws IOException {
- if (certType == CertificateStore.CertType.VALID_CERTS) {
- return validCertsTable.iterator();
- }
-
- if (certType == CertificateStore.CertType.REVOKED_CERTS) {
- return revokedCertsTable.iterator();
- }
-
- return null;
- }
-
@Override
public Table<PipelineID, Pipeline> getPipelineTable() {
return pipelineTable;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
index 3fb32a2e26..88d95d1c82 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -25,19 +25,13 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetAll
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto.ResponseCode;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCrlsResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetLatestCrlIdResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetSCMCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMListCertificateResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRemoveExpiredCertificatesResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMRevokeCertificatesResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Status;
@@ -45,7 +39,6 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ha.RatisUtil;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
@@ -130,21 +123,12 @@ public class SCMSecurityProtocolServerSideTranslatorPB
return scmSecurityResponse.setListCertificateResponseProto(
listCACertificate()).build();
case GetCrls:
- return SCMSecurityResponse.newBuilder()
- .setCmdType(request.getCmdType())
- .setGetCrlsResponseProto(getCrls(request.getGetCrlsRequest()))
- .build();
case GetLatestCrlId:
- return SCMSecurityResponse.newBuilder()
- .setCmdType(request.getCmdType())
- .setGetLatestCrlIdResponseProto(getLatestCrlId(
- request.getGetLatestCrlIdRequest()))
- .build();
case RevokeCertificates:
- return SCMSecurityResponse.newBuilder()
- .setCmdType(request.getCmdType())
- .setRevokeCertificatesResponseProto(revokeCertificates(
- request.getRevokeCertificatesRequest()))
+ return scmSecurityResponse
+ .setStatus(Status.INTERNAL_ERROR)
+ .setMessage("Unsupported operation.")
+ .setSuccess(false)
.build();
case GetCert:
return scmSecurityResponse.setGetCertResponseProto(
@@ -326,7 +310,7 @@ public class SCMSecurityProtocolServerSideTranslatorPB
public SCMListCertificateResponseProto listCertificate(
SCMListCertificateRequestProto request) throws IOException {
List<String> certs = impl.listCertificate(request.getRole(),
- request.getStartCertId(), request.getCount(), request.getIsRevoked());
+ request.getStartCertId(), request.getCount());
SCMListCertificateResponseProto.Builder builder =
SCMListCertificateResponseProto
@@ -338,40 +322,6 @@ public class SCMSecurityProtocolServerSideTranslatorPB
}
- public SCMGetCrlsResponseProto getCrls(
- SCMGetCrlsRequestProto request) throws IOException {
- List<CRLInfo> crls = impl.getCrls(request.getCrlIdList());
- SCMGetCrlsResponseProto.Builder builder =
- SCMGetCrlsResponseProto.newBuilder();
- for (CRLInfo crl : crls) {
- try {
- builder.addCrlInfos(crl.getProtobuf());
- } catch (SCMSecurityException e) {
- LOG.error("Fail in parsing CRL info", e);
- throw new SCMSecurityException("Fail in parsing CRL info", e);
- }
- }
- return builder.build();
- }
-
- public SCMGetLatestCrlIdResponseProto getLatestCrlId(
- SCMGetLatestCrlIdRequestProto request) throws IOException {
- SCMGetLatestCrlIdResponseProto.Builder builder =
- SCMGetLatestCrlIdResponseProto
- .newBuilder().
- setCrlId(impl.getLatestCrlId());
- return builder.build();
- }
-
- public SCMRevokeCertificatesResponseProto revokeCertificates(
- SCMRevokeCertificatesRequestProto request) throws IOException {
- SCMRevokeCertificatesResponseProto.Builder builder =
- SCMRevokeCertificatesResponseProto.newBuilder().setCrlId(
- impl.revokeCertificates(request.getCertIdsList(),
- request.getReason().getNumber(), request.getRevokeTime()));
- return builder.build();
- }
-
public SCMGetCertResponseProto getRootCACertificate() throws IOException {
if (scm.getScmStorageConfig().checkPrimarySCMIdInitialized()) {
throw createNotHAException();
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java
deleted file mode 100644
index 69cb9a7c34..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/CRLStatusReportHandler.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.security;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatusReportFromDatanode;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.crl.CRLStatus;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Handles CRL Status Reports from datanode.
- */
-public class CRLStatusReportHandler implements
- EventHandler<CRLStatusReportFromDatanode> {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(CRLStatusReportHandler.class);
- private CertificateStore certStore = null;
- private final boolean isSecurityEnabled;
-
- public CRLStatusReportHandler(CertificateStore certificateStore,
- OzoneConfiguration conf) {
- isSecurityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
- if (isSecurityEnabled) {
- Preconditions.checkNotNull(certificateStore);
- this.certStore = certificateStore;
- }
- }
-
- @Override
- public void onMessage(CRLStatusReportFromDatanode reportFromDatanode,
- EventPublisher publisher) {
- if (isSecurityEnabled) {
- Preconditions.checkNotNull(reportFromDatanode);
- DatanodeDetails dn = reportFromDatanode.getDatanodeDetails();
- Preconditions.checkNotNull(dn, "CRLStatusReport is "
- + "missing DatanodeDetails.");
-
- if (LOGGER.isTraceEnabled()) {
- LOGGER.trace("Processing CRL status report for dn: {}", dn);
- }
- CRLStatusReport crlStatusReport = reportFromDatanode.getReport();
- long receivedCRLId = crlStatusReport.getReceivedCrlId();
- List<Long> pendingCRLIds = crlStatusReport.getPendingCrlIdsList();
-
- if (LOGGER.isTraceEnabled()) {
- LOGGER.trace("Updating Processed CRL Id: {} and Pending CRL Ids: {} ",
- receivedCRLId,
- pendingCRLIds);
- }
-
- CRLStatus crlStatus = new CRLStatus(receivedCRLId, pendingCRLIds);
- certStore.setCRLStatusForDN(dn.getUuid(), crlStatus);
-
- // Todo: send command for new CRL
- // if crl > dn received crl id, then send a command to DN to process the
- // new CRL via heartbeat response.
-
- }
- }
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
index c46e6e8b1d..8c1f7e22bf 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
@@ -72,7 +72,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NEW_KEY_CERT_DIR_NAME_P
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DIR_NAME_DEFAULT;
import static org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator.CERTIFICATE_ID;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME;
/**
@@ -692,7 +691,7 @@ public class RootCARotationManager extends StatefulService {
X509Certificate rootCACert = null;
try {
if (scm.getCertificateStore().getCertificateByID(
- rootCACertHolder.getSerialNumber(), VALID_CERTS) == null) {
+ rootCACertHolder.getSerialNumber()) == null) {
LOG.info("Persist root certificate {} to cert store",
rootCACertId);
rootCACert =
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
index f2a753f9da..6a849ca7db 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
@@ -22,46 +22,29 @@ package org.apache.hadoop.hdds.scm.server;
import java.io.IOException;
import java.lang.reflect.Proxy;
import java.math.BigInteger;
-import java.security.cert.CRLException;
-import java.security.cert.X509CRL;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Date;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol;
-import org.apache.hadoop.hdds.security.x509.crl.CRLStatus;
import org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CRLApprover;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.X509v2CRLBuilder;
-import org.bouncycastle.operator.OperatorCreationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.SCM;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
/**
* A Certificate Store class that persists certificates issued by SCM CA.
@@ -71,14 +54,10 @@ public final class SCMCertStore implements CertificateStore {
LoggerFactory.getLogger(SCMCertStore.class);
private SCMMetadataStore scmMetadataStore;
private final Lock lock;
- private final AtomicLong crlSequenceId;
- private final Map<UUID, CRLStatus> crlStatusMap;
- private SCMCertStore(SCMMetadataStore dbStore, long sequenceId) {
+ private SCMCertStore(SCMMetadataStore dbStore) {
this.scmMetadataStore = dbStore;
lock = new ReentrantLock();
- crlSequenceId = new AtomicLong(sequenceId);
- crlStatusMap = new ConcurrentHashMap<>();
}
@Override
@@ -129,8 +108,7 @@ public final class SCMCertStore implements CertificateStore {
public void checkValidCertID(BigInteger serialID) throws IOException {
lock.lock();
try {
- if ((getCertificateByID(serialID, VALID_CERTS) != null) ||
- (getCertificateByID(serialID, CertType.REVOKED_CERTS) != null)) {
+ if (getCertificateByID(serialID) != null) {
throw new SCMSecurityException("Conflicting certificate ID" + serialID);
}
} finally {
@@ -138,89 +116,6 @@ public final class SCMCertStore implements CertificateStore {
}
}
- @Override
- public Optional<Long> revokeCertificates(
- List<BigInteger> serialIDs,
- X509CertificateHolder caCertificateHolder,
- CRLReason reason,
- Date revocationTime,
- CRLApprover crlApprover)
- throws IOException {
- Date now = new Date();
- X509v2CRLBuilder builder =
- new X509v2CRLBuilder(caCertificateHolder.getIssuer(), now);
- List<X509Certificate> certsToRevoke = new ArrayList<>();
- X509CRL crl;
- Optional<Long> sequenceId = Optional.empty();
- lock.lock();
- try {
- for (BigInteger serialID: serialIDs) {
- X509Certificate cert =
- getCertificateByID(serialID, CertType.VALID_CERTS);
- if (cert == null && LOG.isWarnEnabled()) {
- LOG.warn("Trying to revoke a certificate that is not valid. " +
- "Serial ID: {}", serialID.toString());
- } else if (getCertificateByID(serialID, CertType.REVOKED_CERTS)
- != null) {
- LOG.warn("Trying to revoke a certificate that is already revoked.");
- } else {
- builder.addCRLEntry(serialID, revocationTime,
- reason.getValue().intValue());
- certsToRevoke.add(cert);
- }
- }
- if (!certsToRevoke.isEmpty()) {
- try {
- crl = crlApprover.sign(builder);
- } catch (OperatorCreationException | CRLException e) {
- throw new SCMSecurityException("Unable to create Certificate " +
- "Revocation List.", e);
- }
- // let us do this in a transaction.
- try (BatchOperation batch =
- scmMetadataStore.getStore().initBatchOperation()) {
- // Move the certificates from Valid Certs table to Revoked Certs Table
- // only if the revocation time has passed.
- if (now.after(revocationTime) || now.equals(revocationTime)) {
- for (X509Certificate cert : certsToRevoke) {
- CertInfo certInfo = new CertInfo.Builder()
- .setX509Certificate(cert)
- .setTimestamp(now.getTime())
- .build();
- scmMetadataStore.getRevokedCertsV2Table()
- .putWithBatch(batch, cert.getSerialNumber(), certInfo);
- scmMetadataStore.getValidCertsTable()
- .deleteWithBatch(batch, cert.getSerialNumber());
- }
- }
- long id = crlSequenceId.incrementAndGet();
- CRLInfo crlInfo = new CRLInfo.Builder()
- .setX509CRL(crl)
- .setCreationTimestamp(now.getTime())
- .setCrlSequenceID(id)
- .build();
- scmMetadataStore.getCRLInfoTable().putWithBatch(
- batch, id, crlInfo);
-
- // Update the CRL Sequence Id Table with the last sequence id.
- scmMetadataStore.getCRLSequenceIdTable().putWithBatch(batch,
- CRL_SEQUENCE_ID_KEY, id);
- scmMetadataStore.getStore().commitBatchOperation(batch);
- sequenceId = Optional.of(id);
- }
- }
- } finally {
- lock.unlock();
- }
- return sequenceId;
- }
-
- @Override
- public void removeExpiredCertificate(BigInteger serialID)
- throws IOException {
- // TODO: Later this allows removal of expired certificates from the system.
- }
-
@Override
public List<X509Certificate> removeAllExpiredCertificates()
throws IOException {
@@ -260,26 +155,14 @@ public final class SCMCertStore implements CertificateStore {
}
@Override
- public X509Certificate getCertificateByID(BigInteger serialID,
- CertType certType)
+ public X509Certificate getCertificateByID(BigInteger serialID)
throws IOException {
- if (certType == VALID_CERTS) {
- return scmMetadataStore.getValidCertsTable().get(serialID);
- } else {
- CertInfo certInfo = getRevokedCertificateInfoByID(serialID);
- return certInfo != null ? certInfo.getX509Certificate() : null;
- }
- }
-
- @Override
- public CertInfo getRevokedCertificateInfoByID(BigInteger serialID)
- throws IOException {
- return scmMetadataStore.getRevokedCertsV2Table().get(serialID);
+ return scmMetadataStore.getValidCertsTable().get(serialID);
}
@Override
public List<X509Certificate> listCertificate(NodeType role,
- BigInteger startSerialID, int count, CertType certType)
+ BigInteger startSerialID, int count)
throws IOException {
List<X509Certificate> results = new ArrayList<>();
String errorMessage = "Fail to list certificate from SCM metadata store";
@@ -289,34 +172,13 @@ public final class SCMCertStore implements CertificateStore {
startSerialID = null;
}
- if (certType == VALID_CERTS) {
- List<? extends Table.KeyValue<BigInteger, X509Certificate>> certs =
- getValidCertTableList(role, startSerialID, count);
-
- for (Table.KeyValue<BigInteger, X509Certificate> kv : certs) {
- try {
- X509Certificate cert = kv.getValue();
- results.add(cert);
- } catch (IOException e) {
- LOG.error(errorMessage, e);
- throw new SCMSecurityException(errorMessage);
- }
- }
- } else {
- List<? extends Table.KeyValue<BigInteger, CertInfo>> certs =
- scmMetadataStore.getRevokedCertsV2Table().getRangeKVs(
- startSerialID, count, null);
-
- for (Table.KeyValue<BigInteger, CertInfo> kv : certs) {
- try {
- CertInfo certInfo = kv.getValue();
- X509Certificate cert = certInfo != null ?
- certInfo.getX509Certificate() : null;
- results.add(cert);
- } catch (IOException e) {
- LOG.error(errorMessage, e);
- throw new SCMSecurityException(errorMessage);
- }
+ for (Table.KeyValue<BigInteger, X509Certificate> kv : getValidCertTableList(role, startSerialID, count)) {
+ try {
+ X509Certificate cert = kv.getValue();
+ results.add(cert);
+ } catch (IOException e) {
+ LOG.error(errorMessage, e);
+ throw new SCMSecurityException(errorMessage);
}
}
return results;
@@ -353,7 +215,6 @@ public final class SCMCertStore implements CertificateStore {
public static class Builder {
private SCMMetadataStore metadataStore;
- private long crlSequenceId;
private SCMRatisServer scmRatisServer;
@@ -362,19 +223,14 @@ public final class SCMCertStore implements CertificateStore {
return this;
}
- public Builder setCRLSequenceId(long sequenceId) {
- this.crlSequenceId = sequenceId;
- return this;
- }
-
public Builder setRatisServer(final SCMRatisServer ratisServer) {
scmRatisServer = ratisServer;
return this;
}
public CertificateStore build() {
- final SCMCertStore scmCertStore = new SCMCertStore(metadataStore,
- crlSequenceId);
+ final SCMCertStore scmCertStore = new SCMCertStore(metadataStore
+ );
final SCMHAInvocationHandler scmhaInvocationHandler =
new SCMHAInvocationHandler(SCMRatisProtocol.RequestType.CERT_STORE,
@@ -386,37 +242,4 @@ public final class SCMCertStore implements CertificateStore {
}
}
-
- @Override
- public List<CRLInfo> getCrls(List<Long> crlIds) throws IOException {
- List<CRLInfo> results = new ArrayList<>();
- for (Long crlId : crlIds) {
- try {
- CRLInfo crlInfo =
- scmMetadataStore.getCRLInfoTable().get(crlId);
- results.add(crlInfo);
- } catch (IOException e) {
- LOG.error("Fail to get CRLs from SCM metadata store for crlId: "
- + crlId, e);
- throw new SCMSecurityException("Fail to get CRLs from SCM metadata " +
- "store for crlId: " + crlId, e);
- }
- }
- return results;
- }
-
- @Override
- public long getLatestCrlId() {
- return crlSequenceId.get();
- }
-
- @Override
- public CRLStatus getCRLStatusForDN(UUID uuid) {
- return crlStatusMap.get(uuid);
- }
-
- @Override
- public void setCRLStatusForDN(UUID uuid, CRLStatus crlStatus) {
- crlStatusMap.put(uuid, crlStatus);
- }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index 484a1e6f0f..cca2df0037 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
@@ -441,16 +439,4 @@ public final class SCMDatanodeHeartbeatDispatcher {
super(datanodeDetails, report);
}
}
-
- /**
- * CRL Status report event payload with origin.
- */
- public static class CRLStatusReportFromDatanode
- extends ReportFromDatanode<CRLStatusReport> {
-
- public CRLStatusReportFromDatanode(DatanodeDetails datanodeDetails,
- CRLStatusReport report) {
- super(datanodeDetails, report);
- }
- }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index 40431330d1..6fea1f106f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -20,21 +20,17 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
import java.io.IOException;
-import java.math.BigInteger;
import java.net.InetSocketAddress;
import java.security.cert.CertPath;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
-import java.util.Date;
import java.util.List;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-import java.util.stream.Collectors;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -51,12 +47,8 @@ import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolDatanodePB;
import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolOmPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.protocolPB.SecretKeyProtocolScmPB;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.protocol.SecretKeyProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdds.scm.update.server.SCMUpdateServiceGrpcServer;
-import org.apache.hadoop.hdds.scm.update.client.UpdateServiceConfig;
-import org.apache.hadoop.hdds.scm.update.server.SCMCRLStore;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.security.exception.SCMSecretKeyException;
@@ -64,7 +56,6 @@ import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey;
import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -78,7 +69,6 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.UserGroupInformation;
-import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,7 +100,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
private CertificateServer rootCertificateServer;
private final CertificateServer scmCertificateServer;
private final RPC.Server rpcServer; // HADOOP RPC SERVER
- private final SCMUpdateServiceGrpcServer grpcUpdateServer; // gRPC SERVER
private final InetSocketAddress rpcAddress;
private final ProtocolMessageMetrics metrics;
private final ProtocolMessageMetrics secretKeyMetrics;
@@ -179,10 +168,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
rpcServer.refreshServiceAcl(conf, SCMPolicyProvider.getInstance());
}
-
- this.grpcUpdateServer = new SCMUpdateServiceGrpcServer(
- conf.getObject(UpdateServiceConfig.class),
- new SCMCRLStore(scmCertificateServer));
}
/**
@@ -439,28 +424,23 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
}
/**
- *
- * @param role - node role: OM/SCM/DN.
- * @param startSerialId - start certificate serial id.
- * @param count - max number of certificates returned in a batch.
- * @param isRevoked - whether list for revoked certs only.
+ * @param role - node role: OM/SCM/DN.
+ * @param startSerialId - start certificate serial id.
+ * @param count - max number of certificates returned in a batch.
* @return
* @throws IOException
*/
@Override
public List<String> listCertificate(NodeType role,
- long startSerialId, int count, boolean isRevoked) throws IOException {
- List<X509Certificate> certificates =
- scmCertificateServer.listCertificate(role, startSerialId, count,
- isRevoked);
+ long startSerialId, int count) throws IOException {
+ List<X509Certificate> certificates = scmCertificateServer.listCertificate(role, startSerialId, count);
List<String> results = new ArrayList<>(certificates.size());
for (X509Certificate cert : certificates) {
try {
String certStr = getPEMEncodedString(cert);
results.add(certStr);
} catch (SCMSecurityException e) {
- throw new SCMSecurityException("listCertificate operation failed.",
- e, e.getErrorCode());
+ throw new SCMSecurityException("listCertificate operation failed.", e, e.getErrorCode());
}
}
return results;
@@ -469,7 +449,7 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
@Override
public List<String> listCACertificate() throws IOException {
List<String> caCerts =
- listCertificate(NodeType.SCM, 0, 10, false);
+ listCertificate(NodeType.SCM, 0, 10);
return caCerts;
}
@@ -490,36 +470,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
scmCertificateClient.getCACertificate());
}
- @Override
- public List<CRLInfo> getCrls(List<Long> crlIds) throws IOException {
- return scmCertificateServer.getCrls(crlIds);
- }
-
- @Override
- public long getLatestCrlId() {
- return scmCertificateServer.getLatestCrlId();
- }
-
- @Override
- public long revokeCertificates(List<String> certIds, int reason,
- long revocationTime) throws IOException {
- storageContainerManager.checkAdminAccess(getRpcRemoteUser(), false);
-
- Future<Optional<Long>> revoked = scmCertificateServer.revokeCertificates(
- certIds.stream().map(id -> new BigInteger(id))
- .collect(Collectors.toList()), CRLReason.lookup(reason),
- new Date(revocationTime));
- try {
- Long crlId = revoked.get().get();
- getGrpcUpdateServer().notifyCrlUpdate();
- return crlId;
- } catch (InterruptedException | ExecutionException e) {
- Thread.currentThread().interrupt();
- throw new SCMException("Fail to revoke certs",
- ResultCodes.FAILED_TO_REVOKE_CERTIFICATES);
- }
- }
-
@Override
public List<String> removeExpiredCertificates() throws IOException {
storageContainerManager.checkAdminAccess(getRpcRemoteUser(), false);
@@ -535,10 +485,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
return String.valueOf(sequenceIdGen.getNextId(CERTIFICATE_ID));
}
- public SCMUpdateServiceGrpcServer getGrpcUpdateServer() {
- return grpcUpdateServer;
- }
-
@VisibleForTesting
public UserGroupInformation getRpcRemoteUser() {
return Server.getRemoteUser();
@@ -558,7 +504,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
LOGGER.info(startupMsg);
metrics.register();
getRpcServer().start();
- getGrpcUpdateServer().start();
}
public void stop() {
@@ -566,7 +511,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
LOGGER.info("Stopping the SCMSecurityProtocolServer.");
metrics.unregister();
getRpcServer().stop();
- getGrpcUpdateServer().stop();
} catch (Exception ex) {
LOGGER.error("SCMSecurityProtocolServer stop failed.", ex);
}
@@ -576,7 +520,6 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol,
LOGGER.info("Join RPC server for SCMSecurityProtocolServer.");
getRpcServer().join();
LOGGER.info("Join gRPC server for SCMSecurityProtocolServer.");
- getGrpcUpdateServer().join();
}
public synchronized CertificateServer getRootCertificateServer() {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index e86dab5fd7..444b341990 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingO
import org.apache.hadoop.hdds.scm.container.replication.DatanodeCommandCountUpdatedHandler;
import org.apache.hadoop.hdds.scm.container.replication.LegacyReplicationManager;
import org.apache.hadoop.hdds.scm.ha.SCMServiceException;
-import org.apache.hadoop.hdds.scm.security.CRLStatusReportHandler;
import org.apache.hadoop.hdds.scm.ha.BackgroundSCMService;
import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
@@ -203,12 +202,10 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_EX
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_QUEUE_WAIT_THRESHOLD_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static org.apache.hadoop.hdds.scm.security.SecretKeyManagerService.isSecretKeyEnable;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READONLY_ADMINISTRATORS;
-import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME;
import static org.apache.hadoop.security.UserGroupInformation.getCurrentUser;
@@ -503,8 +500,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmNodeManager, containerManager, scmContext);
PipelineActionHandler pipelineActionHandler =
new PipelineActionHandler(pipelineManager, scmContext, configuration);
- CRLStatusReportHandler crlStatusReportHandler =
- new CRLStatusReportHandler(certificateStore, configuration);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
@@ -577,7 +572,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
(DeletedBlockLogImpl) scmBlockManager.getDeletedBlockLog());
eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
- eventQueue.addHandler(SCMEvents.CRL_STATUS_REPORT, crlStatusReportHandler);
scmNodeManager.registerSendCommandNotify(
SCMCommandProto.Type.deleteBlocksCommand,
@@ -879,8 +873,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
certificateStore =
new SCMCertStore.Builder().setMetadaStore(scmMetadataStore)
- .setRatisServer(scmHAManager.getRatisServer())
- .setCRLSequenceId(getLastSequenceIdForCRL()).build();
+ .setRatisServer(scmHAManager.getRatisServer()).build();
final CertificateServer scmCertificateServer;
@@ -957,16 +950,14 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
// and ratis server initialized with statemachine. We need to do only
// for primary scm, for other bootstrapped scm's certificates will be
// persisted via ratis.
- if (certificateStore.getCertificateByID(certSerial,
- VALID_CERTS) == null) {
+ if (certificateStore.getCertificateByID(certSerial) == null) {
LOG.info("Storing sub-ca certificate serialId {} on primary SCM",
certSerial);
certificateStore.storeValidScmCertificate(
certSerial, scmCertificateClient.getCertificate());
}
X509Certificate rootCACert = scmCertificateClient.getCACertificate();
- if (certificateStore.getCertificateByID(rootCACert.getSerialNumber(),
- VALID_CERTS) == null) {
+ if (certificateStore.getCertificateByID(rootCACert.getSerialNumber()) == null) {
LOG.info("Storing root certificate serialId {}",
rootCACert.getSerialNumber());
certificateStore.storeValidScmCertificate(
@@ -1076,19 +1067,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
}
- long getLastSequenceIdForCRL() throws IOException {
- Long sequenceId =
- scmMetadataStore.getCRLSequenceIdTable().get(CRL_SEQUENCE_ID_KEY);
- // If the CRL_SEQUENCE_ID_KEY does not exist in DB return 0 so that new
- // CRL requests can have sequence id starting from 1.
- if (sequenceId == null) {
- return 0L;
- }
- // If there exists a last sequence id in the DB, the new incoming
- // CRL requests must have sequence ids greater than the one stored in the DB
- return sequenceId;
- }
-
/**
* Builds a message for logging startup information about an RPC server.
*
@@ -1621,8 +1599,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
for (String cert : pemEncodedCerts) {
X509Certificate x509Certificate = CertificateCodec.getX509Certificate(
cert, CertificateCodec::toIOException);
- if (certificateStore.getCertificateByID(
- x509Certificate.getSerialNumber(), VALID_CERTS) == null) {
+ if (certificateStore.getCertificateByID(x509Certificate.getSerialNumber()) == null) {
LOG.info("Persist certificate serialId {} on Scm Bootstrap Node " +
"{}", x509Certificate.getSerialNumber(),
scmStorageConfig.getScmId());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLStore.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLStore.java
deleted file mode 100644
index 89775c419c..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLStore.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.scm.update.client.CRLStore;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Class for SCM CRL store.
- */
-public class SCMCRLStore implements CRLStore {
-
- private final CertificateServer certServer;
-
- public SCMCRLStore(CertificateServer certServer) {
- this.certServer = certServer;
- }
-
- @Override
- public long getLatestCrlId() {
- return certServer.getLatestCrlId();
- }
-
- @Override
- public CRLInfo getCRL(long crlId) throws IOException {
- List<Long> crlIdList = new ArrayList<>();
- crlIdList.add(crlId);
- return certServer.getCrls(crlIdList).get(0);
- }
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLUpdateHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLUpdateHandler.java
deleted file mode 100644
index f4eefab26e..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLUpdateHandler.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.CRLUpdateResponse;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-import org.apache.hadoop.hdds.scm.update.client.CRLStore;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.ratis.thirdparty.io.grpc.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Class handle the CRL client update and response.
- */
-public class SCMCRLUpdateHandler implements SCMUpdateHandler {
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMCRLUpdateHandler.class);
- private final CRLStore crlStore;
-
- private static final SCMUpdateServiceProtos.Type TYPE =
- SCMUpdateServiceProtos.Type.CRLUpdate;
-
- private final Map<UUID, CRLClientInfo> clients;
-
- SCMCRLUpdateHandler(CRLStore crlStore) {
- this.crlStore = crlStore;
- clients = new ConcurrentHashMap<>();
- }
-
- public SCMUpdateServiceProtos.Type getType() {
- return TYPE;
- }
-
- @Override
- public void handleClientRequest(SCMUpdateServiceProtos.UpdateRequest request,
- SCMUpdateClientInfo clientInfo) {
- SCMUpdateServiceProtos.CRLUpdateRequest updateStatusRequest =
- request.getCrlUpdateRequest();
- long clientCrlId = updateStatusRequest.getReceivedCrlId();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Client {} updateStatus \nclientCrlId {} \npendingCrls {}",
- clientInfo.getClientId(), clientCrlId,
- updateStatusRequest.getPendingCrlIdsList().toString());
- }
-
- CRLClientInfo crlClientInfo;
- if (!clients.containsKey(clientInfo.getClientId())) {
- crlClientInfo = new CRLClientInfo(clientInfo);
- clients.put(clientInfo.getClientId(), crlClientInfo);
- } else {
- crlClientInfo = clients.get(clientInfo.getClientId());
- }
-
- crlClientInfo.setPendingCrlIds(
- request.getCrlUpdateRequest().getPendingCrlIdsList());
- crlClientInfo.setReceivedCrlId(
- request.getCrlUpdateRequest().getReceivedCrlId());
-
- sendCrlUpdateToClient(crlClientInfo);
- }
-
- @Override
- public void onUpdate() {
- LOG.debug("Update due to certificate revocation");
- // server crl id is usually > client crl id when this is invoked.
- clients.values().forEach(client -> {
- sendCrlUpdateToClient(client);
- });
- }
-
- @Override
- public void onRemoveClient(SCMUpdateClientInfo clientInfo) {
- clients.remove(clientInfo.getClientId());
- }
-
- private void sendCrlUpdateToClient(CRLClientInfo client) {
- long clientCrlId = client.getReceivedCrlId();
- long serverCrlId = crlStore.getLatestCrlId();
-
- if (clientCrlId >= serverCrlId) {
- return;
- }
-
- LOG.debug("## Server: clientCrlId {} serverCrlId {}",
- clientCrlId, serverCrlId);
-
- long nextCrlId = clientCrlId + 1;
- try {
- CRLInfo crlInfo = null;
- while (crlInfo == null && nextCrlId <= serverCrlId) {
- crlInfo = crlStore.getCRL(nextCrlId);
- nextCrlId++;
- }
- if (crlInfo == null) {
- LOG.debug("Nothing to send to client");
- return;
- }
- sendCrlToClient(crlInfo, client.getUpdateClientInfo());
- } catch (Exception e) {
- LOG.error("Failed to handle client update.", e);
- client.getUpdateClientInfo().getResponseObserver().onError(Status.INTERNAL
- .withDescription("Failed to send crl" + nextCrlId +
- " to client " + client.getUpdateClientInfo().getClientId())
- .asException());
- }
- }
-
- private void sendCrlToClient(CRLInfo crl, SCMUpdateClientInfo clientInfo)
- throws SCMSecurityException {
- LOG.debug("Sending client# {} with crl: {} ",
- clientInfo.getClientId(), crl.getCrlSequenceID());
- clientInfo.getResponseObserver().onNext(UpdateResponse.newBuilder()
- .setUpdateType(SCMUpdateServiceProtos.Type.CRLUpdate)
- .setCrlUpdateResponse(
- CRLUpdateResponse.newBuilder().setCrlInfo(crl.getCRLProto3()))
- .build());
- }
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientManager.java
deleted file mode 100644
index 710cd20871..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientManager.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.ratis.thirdparty.io.grpc.Status;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.Type;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Class that manages SCM update clients.
- */
-public class SCMUpdateClientManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMUpdateClientManager.class);
- private Map<UUID, SCMUpdateClientInfo> clients;
- private Map<Type, SCMUpdateHandler> handlers;
-
- public SCMUpdateClientManager() {
- clients = new ConcurrentHashMap<>();
- handlers = new ConcurrentHashMap<>();
- }
-
- public void registerHandler(SCMUpdateHandler handler) {
- handlers.put(handler.getType(), handler);
- }
-
- public void unRegisterHandler(Type type) {
- handlers.remove(type);
- }
-
- public UUID addClient() throws SCMException {
- UUID clientId = UUID.randomUUID();
- int retryCount = 5;
- while (clients.containsKey(clientId)) {
- if (retryCount > 0) {
- clientId = UUID.randomUUID();
- retryCount--;
- } else {
- throw new SCMException("Failed to add CRL client with random clientId" +
- " collision", SCMException.ResultCodes.FAILED_TO_ADD_CRL_CLIENT);
- }
- }
-
- SCMUpdateClientInfo clientInfo = new SCMUpdateClientInfo(clientId);
- clients.put(clientId, clientInfo);
- return clientId;
- }
-
- // this does not necessarily produce a server response via responseObserver.
- public void handleClientUpdate(UpdateRequest request,
- StreamObserver<UpdateResponse> responseObserver) {
- UUID clientId = SCMUpdateClientInfo.fromClientIdProto(
- request.getClientId());
-
- // Unknown client update
- if (!clients.containsKey(clientId)) {
- responseObserver.onError(Status.INVALID_ARGUMENT
- .withDescription("Client must subscribe before it can " +
- "send/receive updates")
- .asException());
- }
-
- // record the server to client channel
- SCMUpdateClientInfo clientInfo = clients.get(clientId);
- if (clientInfo.getResponseObserver() == null) {
- clientInfo.setResponseObserver(responseObserver);
- }
-
- if (handlers.containsKey(request.getUpdateType())) {
- handlers.get(request.getUpdateType())
- .handleClientRequest(request, clientInfo);
- } else {
- responseObserver.onError(Status.INVALID_ARGUMENT
- .withDescription("Unknown client update type.")
- .asException());
- }
- }
-
- /**
- * Remove client by client Id.
- * @param clientId - client Id
- * @return true if client is removed, false otherwise.
- */
- public boolean removeClient(UUID clientId) {
- if (clients.containsKey(clientId)) {
- SCMUpdateClientInfo clientInfo = clients.remove(clientId);
- handlers.values().forEach(handler -> handler.onRemoveClient(clientInfo));
- LOG.info("Client {} removed.", clientId);
- return true;
- }
- return false;
- }
-
- /**
- * Remove client by its responseObserver obj.
- * @param responseObserver - response observer of the client
- * @return true if client is removed, false otherwise.
- */
- public boolean removeClient(StreamObserver<UpdateResponse> responseObserver) {
- UUID clientId = null;
- for (SCMUpdateClientInfo client : clients.values()) {
- if (client.getResponseObserver() == responseObserver) {
- clientId = client.getClientId();
- break;
- }
- }
- if (clientId != null) {
- LOG.debug("Remove client {} by responseObserver", clientId);
- removeClient(clientId);
- return true;
- }
- LOG.debug("Remove client {} by responseObserver not found!");
- return false;
- }
-
- public void onUpdate(Type type) {
- if (handlers.containsKey(type)) {
- handlers.get(type).onUpdate();
- } else {
- LOG.warn("Unknown update type to broadcast!");
- }
- }
-}
-
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateHandler.java
deleted file mode 100644
index 026bd4ad05..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateHandler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.Type;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateRequest;
-
-/**
- * Interface used by server side to handle client update and server publish.
- */
-public interface SCMUpdateHandler {
-
- /**
- * handle client update request.
- * @param request
- * @param clientInfo
- */
- void handleClientRequest(UpdateRequest request,
- SCMUpdateClientInfo clientInfo);
-
- /**
- * Handle server broadcast to all clients as needed.
- */
- void onUpdate();
-
- /**
- * Handle server remove client due to error streaming to the client.
- * @param clientInfo
- */
- void onRemoveClient(SCMUpdateClientInfo clientInfo);
-
- /**
- * Return the type of Update the handler can handle.
- * @return the type of Update the handler can handle.
- */
- Type getType();
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceGrpcServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceGrpcServer.java
deleted file mode 100644
index 28ae360ce7..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceGrpcServer.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos;
-import org.apache.hadoop.hdds.scm.update.client.CRLStore;
-import org.apache.hadoop.hdds.scm.update.client.UpdateServiceConfig;
-import org.apache.ratis.thirdparty.io.grpc.Server;
-import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * gRPC server for SCM update services.
- */
-public class SCMUpdateServiceGrpcServer {
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMUpdateServiceGrpcServer.class);
-
- private static final String SERVICE_NAME = "SCMUpdateService";
- private CRLStore crlStore;
- private int port;
- private Server server;
- private SCMUpdateServiceImpl scmUpdateService;
- private final AtomicBoolean isStarted = new AtomicBoolean(false);
-
- public SCMUpdateServiceGrpcServer(final UpdateServiceConfig updateConf,
- final CRLStore crlStore) {
- this.crlStore = crlStore;
- this.port = updateConf.getPort();
- }
-
- public int getPort() {
- return this.port;
- }
-
- public void start() throws IOException {
- LOG.info("{} starting", SERVICE_NAME);
- scmUpdateService = new SCMUpdateServiceImpl(crlStore);
- server = ServerBuilder.forPort(port).
- addService(scmUpdateService)
- .build();
-
- if (!isStarted.compareAndSet(false, true)) {
- LOG.info("Ignoring start() since {} has already started.", SERVICE_NAME);
- return;
- } else {
- server.start();
- }
- }
-
- public void stop() {
- LOG.info("{} stopping", SERVICE_NAME);
- if (isStarted.get()) {
- scmUpdateService = null;
- server.shutdown();
- try {
- server.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- LOG.error("failed to shutdown SCMClientGrpcServer", e);
- Thread.currentThread().interrupt();
- } finally {
- server.shutdownNow();
- }
- LOG.info("{} stopped!", SERVICE_NAME);
- isStarted.set(false);
- }
- }
-
- public void join() throws InterruptedException {
- while (isStarted.get()) {
- wait();
- }
- }
-
- public void notifyCrlUpdate() {
- scmUpdateService.notifyUpdate(SCMUpdateServiceProtos.Type.CRLUpdate);
- }
-
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceImpl.java
deleted file mode 100644
index f339c9cac3..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceImpl.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceGrpc;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.SubscribeRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.SubscribeResponse;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.Type;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UpdateResponse;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UnsubscribeRequest;
-import org.apache.hadoop.hdds.protocol.scm.proto.SCMUpdateServiceProtos.UnsubscribeResponse;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.update.client.CRLStore;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.UUID;
-
-/**
- * Impl class for SCM update service.
- * Allows client to subscribe and bi-directional streaming update with server.
- * Currently used for CRL udpate between SCM and OM.
- * Can be extended to update SCM pipeline/container change in future.
- */
-public class SCMUpdateServiceImpl extends
- SCMUpdateServiceGrpc.SCMUpdateServiceImplBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMUpdateServiceImpl.class);
-
- private SCMUpdateClientManager clientManager;
-
- public SCMUpdateServiceImpl(CRLStore crlStore) {
- clientManager = new SCMUpdateClientManager();
- clientManager.registerHandler(new SCMCRLUpdateHandler(crlStore));
- }
-
- @Override
- public void subscribe(SubscribeRequest request,
- StreamObserver<SubscribeResponse> responseObserver) {
- UUID clientId;
- try {
- clientId = clientManager.addClient();
- } catch (SCMException ex) {
- LOG.error("Fail to subscribe for Client.", ex);
- responseObserver.onError(ex);
- return;
- }
- responseObserver.onNext(SubscribeResponse.newBuilder()
- .setClientId(SCMUpdateClientInfo.toClientIdProto(clientId))
- .build());
- responseObserver.onCompleted();
- LOG.info("Client {} subscribed.", clientId);
- }
-
- @Override
- public void unsubscribe(UnsubscribeRequest request,
- StreamObserver<UnsubscribeResponse> responseObserver) {
- UUID clientId = SCMUpdateClientInfo.fromClientIdProto(
- request.getClientId());
- boolean removed = clientManager.removeClient(clientId);
- if (removed) {
- LOG.info("Client {} unsubscribed.", clientId);
- } else {
- LOG.info("Client {} does not exist, no-op for unsubscribe", clientId);
- }
- responseObserver.onNext(UnsubscribeResponse.getDefaultInstance());
- responseObserver.onCompleted();
- }
-
- @Override
- public StreamObserver<UpdateRequest> updateStatus(
- StreamObserver<UpdateResponse> responseObserver) {
- return new StreamObserver<UpdateRequest>() {
- @Override
- public void onNext(UpdateRequest updateRequest) {
- LOG.debug("UpdateStatus onNext");
- clientManager.handleClientUpdate(updateRequest, responseObserver);
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOG.debug("UpdateStatus onError", throwable);
- clientManager.removeClient(responseObserver);
- }
-
- @Override
- public void onCompleted() {
- LOG.debug("UpdateStatus(Client) onComplete");
- responseObserver.onCompleted();
- clientManager.removeClient(responseObserver);
- }
- };
- }
-
- // service prepare a update response and broadcast to all clients subscribed.
- public void notifyUpdate(Type type) {
- clientManager.onUpdate(type);
- }
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java
deleted file mode 100644
index 07c58675e2..0000000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * CRL server package.
- */
-package org.apache.hadoop.hdds.scm.update.server;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
index 6021c33de1..fe5459764c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
@@ -25,8 +25,6 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CRLStatusReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineAction;
import org.apache.hadoop.hdds.protocol.proto
@@ -523,20 +521,6 @@ public final class HddsTestUtils {
return report.build();
}
- /**
- * Create CRL Status report object.
- * @param pendingCRLIds List of Pending CRL Ids in the report.
- * @param receivedCRLId Latest received CRL Id in the report.
- * @return {@link CRLStatusReport}
- */
- public static CRLStatusReport createCRLStatusReport(
- List<Long> pendingCRLIds, long receivedCRLId) {
- CRLStatusReport.Builder report = CRLStatusReport.newBuilder();
- report.addAllPendingCrlIds(pendingCRLIds);
- report.setReceivedCrlId(receivedCRLId);
- return report.build();
- }
-
public static org.apache.hadoop.hdds.scm.container.ContainerInfo
allocateContainer(ContainerManager containerManager)
throws IOException, TimeoutException {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java
deleted file mode 100644
index f2ab42d581..0000000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestCRLStatusReportHandler.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.security;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CRLStatusReport;
-import org.apache.hadoop.hdds.scm.HddsTestUtils;
-import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
-import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl;
-import org.apache.hadoop.hdds.scm.server.SCMCertStore;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CRLStatusReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.crl.CRLStatus;
-import org.apache.hadoop.hdds.server.events.Event;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-
-/**
- * Test for the CRL Status Report Handler.
- */
-public class TestCRLStatusReportHandler implements EventPublisher {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestCRLStatusReportHandler.class);
- private CRLStatusReportHandler crlStatusReportHandler;
- private CertificateStore certificateStore;
- private SCMMetadataStore scmMetadataStore;
-
- @BeforeEach
- public void init(@TempDir Path tempDir) throws IOException {
- OzoneConfiguration config = new OzoneConfiguration();
-
- config.set(HddsConfigKeys.OZONE_METADATA_DIRS,
- tempDir.toAbsolutePath().toString());
- config.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
-
- SCMStorageConfig storageConfig = mock(SCMStorageConfig.class);
- when(storageConfig.getClusterID()).thenReturn("cluster1");
- scmMetadataStore = new SCMMetadataStoreImpl(config);
- certificateStore = new SCMCertStore.Builder()
- .setRatisServer(null)
- .setMetadaStore(scmMetadataStore)
- .build();
- crlStatusReportHandler =
- new CRLStatusReportHandler(certificateStore, config);
- }
-
- @AfterEach
- public void destroyDbStore() throws Exception {
- if (scmMetadataStore.getStore() != null) {
- scmMetadataStore.getStore().close();
- scmMetadataStore = null;
- }
- }
-
- @Test
- public void testCRLStatusReport() {
- DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails();
- DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails();
- List<Long> pendingCRLIds1 = new ArrayList<>();
- List<Long> pendingCRLIds2 = new ArrayList<>();
- pendingCRLIds1.add(3L);
- pendingCRLIds1.add(4L);
- pendingCRLIds2.add(1L);
- CRLStatusReportFromDatanode reportFromDatanode1 =
- getCRLStatusReport(dn1, pendingCRLIds1, 5L);
- CRLStatusReportFromDatanode reportFromDatanode2 =
- getCRLStatusReport(dn2, pendingCRLIds2, 2L);
- crlStatusReportHandler.onMessage(reportFromDatanode1, this);
- CRLStatus crlStatus = certificateStore.getCRLStatusForDN(dn1.getUuid());
- assertThat(crlStatus.getPendingCRLIds()).containsAll(pendingCRLIds1);
- assertEquals(5L, crlStatus.getReceivedCRLId());
-
- pendingCRLIds1.remove(0);
- reportFromDatanode1 = getCRLStatusReport(dn1, pendingCRLIds1, 6L);
- crlStatusReportHandler.onMessage(reportFromDatanode1, this);
- crlStatus = certificateStore.getCRLStatusForDN(dn1.getUuid());
- assertEquals(1, crlStatus.getPendingCRLIds().size());
- assertEquals(4L,
- crlStatus.getPendingCRLIds().get(0).longValue());
- assertEquals(6L, crlStatus.getReceivedCRLId());
-
- crlStatusReportHandler.onMessage(reportFromDatanode2, this);
- crlStatus = certificateStore.getCRLStatusForDN(dn2.getUuid());
- assertThat(crlStatus.getPendingCRLIds()).containsAll(pendingCRLIds2);
- assertEquals(2L, crlStatus.getReceivedCRLId());
- }
-
- private CRLStatusReportFromDatanode getCRLStatusReport(
- DatanodeDetails dn,
- List<Long> pendingCRLIds,
- long receivedCRLId) {
- CRLStatusReport crlStatusReportProto =
- HddsTestUtils.createCRLStatusReport(pendingCRLIds, receivedCRLId);
- return new CRLStatusReportFromDatanode(dn, crlStatusReportProto);
- }
-
- @Override
- public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
- EVENT_TYPE event, PAYLOAD payload) {
- LOG.info("Event is published: {}", payload);
- }
-}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
index 8f69777d04..e57be2f10b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
@@ -24,48 +24,26 @@ import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.CertificateTestUtils;
-import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CRLApprover;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCRLApprover;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyPair;
-import java.security.cert.X509CRLEntry;
import java.security.cert.X509Certificate;
import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
import java.util.List;
-import java.util.Optional;
-import java.util.Set;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.DATANODE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.OM;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.SCM;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
/**
* Test class for @{@link SCMCertStore}.
@@ -73,15 +51,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
public class TestSCMCertStore {
private static final String COMPONENT_NAME = "scm";
- private static final Long INITIAL_SEQUENCE_ID = 1L;
private OzoneConfiguration config;
private SCMMetadataStore scmMetadataStore;
private CertificateStore scmCertStore;
private SecurityConfig securityConfig;
- private X509Certificate x509Certificate;
private KeyPair keyPair;
- private CRLApprover crlApprover;
@BeforeEach
public void setUp(@TempDir Path tempDir) throws Exception {
@@ -95,15 +70,10 @@ public class TestSCMCertStore {
scmMetadataStore = new SCMMetadataStoreImpl(config);
scmCertStore = new SCMCertStore.Builder().setRatisServer(null)
- .setCRLSequenceId(INITIAL_SEQUENCE_ID)
.setMetadaStore(scmMetadataStore)
.build();
Files.createDirectories(securityConfig.getKeyLocation(COMPONENT_NAME));
- x509Certificate = generateX509Cert();
-
- crlApprover = new DefaultCRLApprover(securityConfig,
- keyPair.getPrivate());
}
@AfterEach
@@ -113,177 +83,11 @@ public class TestSCMCertStore {
}
}
- @Test
- public void testRevokeCertificates() throws Exception {
-
- BigInteger serialID = x509Certificate.getSerialNumber();
- scmCertStore.storeValidCertificate(serialID, x509Certificate, SCM);
- Date now = new Date();
-
- assertNotNull(
- scmCertStore.getCertificateByID(serialID,
- VALID_CERTS));
-
- X509CertificateHolder caCertificateHolder =
- new X509CertificateHolder(generateX509Cert().getEncoded());
- List<BigInteger> certs = new ArrayList<>();
- certs.add(x509Certificate.getSerialNumber());
- Optional<Long> sequenceId = scmCertStore.revokeCertificates(certs,
- caCertificateHolder,
- CRLReason.lookup(CRLReason.keyCompromise), now, crlApprover);
-
- assertTrue(sequenceId.isPresent());
- assertEquals(INITIAL_SEQUENCE_ID + 1L, (long) sequenceId.get());
-
- assertNull(
- scmCertStore.getCertificateByID(serialID,
- VALID_CERTS));
-
- CertInfo certInfo = scmCertStore.getRevokedCertificateInfoByID(serialID);
-
- assertNotNull(certInfo);
- assertNotNull(certInfo.getX509Certificate());
- assertThat(certInfo.getTimestamp())
- .withFailMessage("Timestamp should be greater than 0")
- .isGreaterThan(0L);
-
- long crlId = scmCertStore.getLatestCrlId();
- assertEquals(sequenceId.get().longValue(), crlId);
-
- List<CRLInfo> crls = scmCertStore.getCrls(Arrays.asList(crlId));
- assertEquals(1, crls.size());
-
- // CRL Info table should have a CRL with sequence id
- assertNotNull(scmMetadataStore.getCRLInfoTable()
- .get(sequenceId.get()));
-
- // Check the sequence ID table for latest sequence id
- assertEquals(INITIAL_SEQUENCE_ID + 1L, (long)
- scmMetadataStore.getCRLSequenceIdTable().get(CRL_SEQUENCE_ID_KEY));
-
- CRLInfo crlInfo = crls.get(0);
-
- assertEquals(crlInfo.getCrlSequenceID(), sequenceId.get().longValue());
-
- Set<? extends X509CRLEntry> revokedCertificates =
- crlInfo.getX509CRL().getRevokedCertificates();
- assertEquals(1L, revokedCertificates.size());
- assertEquals(x509Certificate.getSerialNumber(),
- revokedCertificates.iterator().next().getSerialNumber());
-
- // Now trying to revoke the already revoked certificate should result in
- // a warning message and no-op. It should not create a new CRL.
- sequenceId = scmCertStore.revokeCertificates(certs,
- caCertificateHolder,
- CRLReason.lookup(CRLReason.unspecified), now, crlApprover);
-
- assertFalse(sequenceId.isPresent());
-
- assertEquals(1L, getTableSize(scmMetadataStore.getCRLInfoTable()));
-
- // Generate 3 more certificates and revoke 2 of them
- List<BigInteger> newSerialIDs = new ArrayList<>();
- for (int i = 0; i < 3; i++) {
- X509Certificate cert = generateX509Cert();
- scmCertStore.storeValidCertificate(cert.getSerialNumber(), cert, SCM);
- newSerialIDs.add(cert.getSerialNumber());
- }
-
- // Add the first 2 certificates to the revocation list
- sequenceId = scmCertStore.revokeCertificates(newSerialIDs.subList(0, 2),
- caCertificateHolder,
- CRLReason.lookup(CRLReason.aACompromise), now, crlApprover);
-
- // This should create a CRL with sequence id INITIAL_SEQUENCE_ID + 2
- // And contain 2 certificates in it
- assertTrue(sequenceId.isPresent());
- assertEquals(sequenceId.get().longValue(),
- scmCertStore.getLatestCrlId());
- assertEquals(INITIAL_SEQUENCE_ID + 2L, (long) sequenceId.get());
-
- // Check the sequence ID table for latest sequence id
- assertEquals(INITIAL_SEQUENCE_ID + 2L, (long)
- scmMetadataStore.getCRLSequenceIdTable().get(CRL_SEQUENCE_ID_KEY));
-
- CRLInfo newCrlInfo = scmCertStore.getCrls(Arrays.asList(
- INITIAL_SEQUENCE_ID + 2)).get(0);
- revokedCertificates = newCrlInfo.getX509CRL().getRevokedCertificates();
- assertEquals(2L, revokedCertificates.size());
- assertNotNull(
- revokedCertificates.stream().filter(c ->
- c.getSerialNumber().equals(newSerialIDs.get(0)))
- .findAny());
-
- assertNotNull(
- revokedCertificates.stream().filter(c ->
- c.getSerialNumber().equals(newSerialIDs.get(1)))
- .findAny());
-
- assertEquals(newCrlInfo.getCrlSequenceID(), sequenceId.get().longValue());
-
- Table<BigInteger, X509Certificate> validCertsTable =
- scmMetadataStore.getValidCertsTable();
- try (TableIterator<BigInteger, ? extends Table.KeyValue<BigInteger,
- X509Certificate>> iterator = validCertsTable.iterator()) {
- assertTrue(iterator.hasNext());
- // Make sure that the last certificate that was not revoked is the one
- // in the valid certs table.
- assertEquals(newSerialIDs.get(2), iterator.next().getKey());
- // Valid certs table should have 1 cert
- assertFalse(iterator.hasNext());
- }
-
- // Revoked certs table should have 3 certs
- assertEquals(3L,
- getTableSize(scmMetadataStore.getRevokedCertsV2Table()));
- }
-
- @Test
- public void testRevokeCertificatesForFutureTime() throws Exception {
- BigInteger serialID = x509Certificate.getSerialNumber();
- scmCertStore.storeValidCertificate(serialID, x509Certificate, SCM);
- Date now = new Date();
- // Set revocation time in the future
- Date revocationTime = new Date(now.getTime() + 500);
-
- X509CertificateHolder caCertificateHolder =
- new X509CertificateHolder(generateX509Cert().getEncoded());
- List<BigInteger> certs = new ArrayList<>();
- certs.add(x509Certificate.getSerialNumber());
- Optional<Long> sequenceId = scmCertStore.revokeCertificates(certs,
- caCertificateHolder,
- CRLReason.lookup(CRLReason.keyCompromise), revocationTime,
- crlApprover);
-
- assertTrue(sequenceId.isPresent());
- assertEquals(INITIAL_SEQUENCE_ID + 1L, (long) sequenceId.get());
-
- assertNotNull(
- scmCertStore.getCertificateByID(serialID,
- VALID_CERTS));
-
- assertNull(
- scmCertStore.getRevokedCertificateInfoByID(serialID));
- }
-
private X509Certificate generateX509Cert() throws Exception {
return KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
"SHA256withRSA");
}
- private long getTableSize(Table<?, ?> table) throws IOException {
- try (TableIterator<?, ?> iterator = table.iterator()) {
- long size = 0;
-
- while (iterator.hasNext()) {
- size++;
- iterator.next();
- }
-
- return size;
- }
- }
-
@Test
public void testGetAndListCertificates() throws Exception {
X509Certificate cert = generateX509Cert();
@@ -340,7 +144,7 @@ public class TestSCMCertStore {
private void checkListCerts(NodeType role, int expected) throws Exception {
List<X509Certificate> certificateList = scmCertStore.listCertificate(role,
- BigInteger.valueOf(0), 10, VALID_CERTS);
+ BigInteger.valueOf(0), 10);
assertEquals(expected, certificateList.size());
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/MockCRLStore.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/MockCRLStore.java
deleted file mode 100644
index 12e8836519..0000000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/MockCRLStore.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
-import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl;
-import org.apache.hadoop.hdds.scm.server.SCMCertStore;
-import org.apache.hadoop.hdds.scm.update.client.CRLStore;
-import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CRLApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCRLApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.bouncycastle.asn1.x509.CRLReason;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.security.KeyPair;
-import java.security.cert.X509Certificate;
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Mock CRL Store impl for test.
- */
-public class MockCRLStore implements CRLStore {
-
- private static final String COMPONENT_NAME = "scm";
- private static final Long INITIAL_SEQUENCE_ID = 0L;
-
- private OzoneConfiguration config;
- private SCMMetadataStore scmMetadataStore;
- private CertificateStore scmCertStore;
- private SecurityConfig securityConfig;
- private KeyPair keyPair;
- private CRLApprover crlApprover;
- private final X509CertificateHolder caCertificateHolder;
- private final Logger log;
-
- public MockCRLStore(Path metadataDir, Logger log) throws Exception {
-
- this.log = log;
- config = new OzoneConfiguration();
- config.set(HddsConfigKeys.OZONE_METADATA_DIRS,
- metadataDir.toAbsolutePath().toString());
-
- securityConfig = new SecurityConfig(config);
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
-
- scmMetadataStore = new SCMMetadataStoreImpl(config);
- scmCertStore = new SCMCertStore.Builder().setRatisServer(null)
- .setCRLSequenceId(INITIAL_SEQUENCE_ID)
- .setMetadaStore(scmMetadataStore)
- .build();
- crlApprover = new DefaultCRLApprover(securityConfig,
- keyPair.getPrivate());
-
- Files.createDirectories(securityConfig.getKeyLocation(COMPONENT_NAME));
- caCertificateHolder = new X509CertificateHolder(generateX509Cert()
- .getEncoded());
- }
-
- public BigInteger issueCert() throws Exception {
- X509Certificate cert = generateX509Cert();
- scmCertStore.storeValidCertificate(cert.getSerialNumber(), cert,
- HddsProtos.NodeType.SCM);
- return cert.getSerialNumber();
- }
-
- public Optional<Long> revokeCert(List<BigInteger> certs,
- Instant revokeTime)
- throws IOException, TimeoutException {
- log.debug("Revoke certs: {}", certs);
- Optional<Long> crlId = scmCertStore.revokeCertificates(certs,
- caCertificateHolder,
- CRLReason.lookup(CRLReason.keyCompromise),
- Date.from(revokeTime), crlApprover);
- List<CRLInfo> crlInfos =
- scmCertStore.getCrls(ImmutableList.of(crlId.get()));
-
- if (!crlInfos.isEmpty()) {
- log.debug("CRL[0]: {}", crlInfos.get(0));
- }
- return crlId;
- }
-
-
- private X509Certificate generateX509Cert() throws Exception {
- return CertificateCodec.getX509Certificate(
- CertificateCodec.getPEMEncodedString(
- KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
- "SHA256withRSA")));
- }
-
- @Override
- public long getLatestCrlId() {
- return scmCertStore.getLatestCrlId();
- }
-
- @Override
- public CRLInfo getCRL(long crlId) throws IOException {
- return scmCertStore.getCrls(Arrays.asList(crlId)).get(0);
- }
-
- public void close() throws Exception {
- if (scmMetadataStore.getStore() != null) {
- scmMetadataStore.getStore().close();
- }
- }
-}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/TestSCMUpdateServiceGrpcServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/TestSCMUpdateServiceGrpcServer.java
deleted file mode 100644
index 09a4078f70..0000000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/TestSCMUpdateServiceGrpcServer.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.update.server;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.update.client.CRLClientUpdateHandler;
-import org.apache.hadoop.hdds.scm.update.client.ClientCRLStore;
-import org.apache.hadoop.hdds.scm.update.client.SCMUpdateClientConfiguration;
-import org.apache.hadoop.hdds.scm.update.client.SCMUpdateServiceGrpcClient;
-import org.apache.hadoop.hdds.scm.update.client.UpdateServiceConfig;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.tag.Unhealthy;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.api.io.TempDir;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.TimeoutException;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-/**
- * Tests for SCM update Service.
- */
-@Timeout(300)
-public class TestSCMUpdateServiceGrpcServer {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSCMUpdateServiceGrpcServer.class);
-
- private MockCRLStore mockCRLStore;
-
- @BeforeEach
- public void setUp(@TempDir Path tempDir) throws Exception {
- mockCRLStore = new MockCRLStore(tempDir, LOG);
- GenericTestUtils.setLogLevel(CRLClientUpdateHandler.getLog(), Level.DEBUG);
- }
-
- @AfterEach
- public void destroyDbStore() throws Exception {
- if (mockCRLStore != null) {
- mockCRLStore.close();
- mockCRLStore = null;
- }
- }
-
- private UpdateServiceConfig getUpdateServiceConfig(OzoneConfiguration conf) {
- return conf.getObject(UpdateServiceConfig.class);
- }
-
- @Test
- public void testStartStop() {
- OzoneConfiguration conf = new OzoneConfiguration();
- SCMUpdateServiceGrpcServer server = new SCMUpdateServiceGrpcServer(
- getUpdateServiceConfig(conf), mockCRLStore);
- ClientCRLStore clientCRLStore = new ClientCRLStore();
- SCMUpdateServiceGrpcClient client =
- new SCMUpdateServiceGrpcClient("localhost", conf, clientCRLStore);
-
- try {
- server.start();
- client.start();
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- // client need to handle the case when the server is stopped first.
- client.stop(true);
- server.stop();
- }
- }
-
-
- @Unhealthy("HDDS-5319")
- @Test
- public void testClientUpdateWithRevoke() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- SCMUpdateServiceGrpcServer server = new SCMUpdateServiceGrpcServer(
- getUpdateServiceConfig(conf), mockCRLStore);
- ClientCRLStore clientCRLStore = new ClientCRLStore();
- SCMUpdateServiceGrpcClient client =
- new SCMUpdateServiceGrpcClient("localhost", conf, clientCRLStore);
- server.start();
- client.start();
-
- try {
- // issue 10 certs
- List<BigInteger> certIds = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- BigInteger certId = mockCRLStore.issueCert();
- certIds.add(certId);
- }
-
- // revoke 4 certs and broadcast
- for (int i = 0; i < 4; i++) {
- revokeCertNow((certIds.get(i)));
- }
- server.notifyCrlUpdate();
-
- GenericTestUtils.waitFor(() -> client.getUpdateCount() == 4, 100, 2000);
- assertEquals(4, client.getUpdateCount());
- assertEquals(0, client.getErrorCount());
-
- revokeCertNow(certIds.get(5));
- server.notifyCrlUpdate();
- GenericTestUtils.waitFor(() -> client.getUpdateCount() > 4, 100, 2000);
- assertEquals(5, client.getUpdateCount());
- assertEquals(0, client.getErrorCount());
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- client.stop(true);
- server.stop();
- }
- }
-
- @Unhealthy("HDDS-5319")
- @Test
- public void testClientUpdateWithDelayedRevoke() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- SCMUpdateServiceGrpcServer server = new SCMUpdateServiceGrpcServer(
- getUpdateServiceConfig(conf), mockCRLStore);
-
- ClientCRLStore clientCRLStore = new ClientCRLStore();
-
- // check pending crl every 5 seconds
- SCMUpdateClientConfiguration updateClientConfiguration =
- conf.getObject(SCMUpdateClientConfiguration.class);
- updateClientConfiguration.setClientCrlCheckInterval(Duration.ofSeconds(2));
- conf.setFromObject(updateClientConfiguration);
-
- SCMUpdateServiceGrpcClient client =
- new SCMUpdateServiceGrpcClient("localhost", conf, clientCRLStore);
- server.start();
- client.start();
-
- try {
- // issue 10 certs
- List<BigInteger> certIds = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- BigInteger certId = mockCRLStore.issueCert();
- certIds.add(certId);
- }
-
- // revoke cert 0
- revokeCertNow((certIds.get(0)));
- server.notifyCrlUpdate();
-
- GenericTestUtils.waitFor(() -> client.getUpdateCount() == 1,
- 100, 2000);
- assertEquals(1, client.getUpdateCount());
- assertEquals(0, client.getErrorCount());
-
- // revoke cert 5 with 10 seconds delay
- revokeCert(certIds.get(5), Instant.now().plus(Duration.ofSeconds(5)));
- server.notifyCrlUpdate();
- GenericTestUtils.waitFor(() -> client.getUpdateCount() > 1,
- 100, 2000);
- assertThat(2L).isLessThanOrEqualTo(client.getUpdateCount());
- assertEquals(0, client.getErrorCount());
- assertThat(1).isGreaterThanOrEqualTo(client.getClientCRLStore()
- .getPendingCrlIds().size());
-
- GenericTestUtils.waitFor(() -> client.getPendingCrlRemoveCount() == 1,
- 100, 20_000);
- assertTrue(client.getClientCRLStore()
- .getPendingCrlIds().isEmpty());
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- client.stop(true);
- server.stop();
- }
- }
-
- private Long revokeCert(BigInteger certId, Instant revokeTime)
- throws IOException, TimeoutException {
- Optional<Long> crlId =
- mockCRLStore.revokeCert(Arrays.asList(certId), revokeTime);
- return crlId.get();
- }
-
- private Long revokeCertNow(BigInteger certId)
- throws IOException, TimeoutException {
- Optional<Long> crlId =
- mockCRLStore.revokeCert(Arrays.asList(certId), Instant.now());
- return crlId.get();
- }
-
- @Unhealthy("HDDS-5319")
- @Test
- public void testClientUpdateWithRestart() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- SCMUpdateServiceGrpcServer server = new SCMUpdateServiceGrpcServer(
- getUpdateServiceConfig(conf), mockCRLStore);
- ClientCRLStore clientCRLStore = new ClientCRLStore();
- SCMUpdateServiceGrpcClient client =
- new SCMUpdateServiceGrpcClient("localhost", conf, clientCRLStore);
- server.start();
- client.start();
-
- try {
- // issue 10 certs
- List<BigInteger> certIds = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- BigInteger certId = mockCRLStore.issueCert();
- certIds.add(certId);
- }
-
- // revoke 4 certs and broadcast
- for (int i = 0; i < 4; i++) {
- revokeCertNow((certIds.get(i)));
- }
- server.notifyCrlUpdate();
- GenericTestUtils.waitFor(() -> client.getUpdateCount() == 4,
- 100, 2000);
- assertEquals(4, client.getUpdateCount());
-
-
- // server restart
- // client onError->
- // 1. reconnect
- // 2. new subscribe resumes from previous state
- LOG.info("Test server restart begin.");
- // server shutdown can lead to duplicate message received on client when
- // client retry connect to the server. The client will handle that.
- server.stop();
- server.start();
- GenericTestUtils.waitFor(() -> client.getErrorCount() == 1,
- 100, 2000);
- assertEquals(4, client.getUpdateCount());
- assertEquals(1, client.getErrorCount());
- assertEquals(4, clientCRLStore.getLatestCrlId());
- LOG.info("Test server restart end.");
-
- revokeCertNow(certIds.get(5));
- server.notifyCrlUpdate();
- GenericTestUtils.waitFor(() -> client.getUpdateCount() > 4,
- 100, 5000);
- assertEquals(5, client.getUpdateCount());
- assertEquals(1, client.getErrorCount());
- assertEquals(5, clientCRLStore.getLatestCrlId());
-
- // client restart
- // server onError->
- // 1. remove stale client
- // 2. new subscribe resumes from previous state.
- LOG.info("Test client restart begin.");
- // a full client channel shutdown and create
- client.stop(true);
- client.createChannel();
- client.start();
- assertEquals(5, clientCRLStore.getLatestCrlId());
- GenericTestUtils.waitFor(() -> client.getUpdateCount() > 5,
- 100, 2000);
- revokeCertNow(certIds.get(6));
- // mostly noop
- server.notifyCrlUpdate();
- LOG.info("Test client restart end.");
-
- GenericTestUtils.waitFor(() -> client.getUpdateCount() > 6,
- 100, 2000);
- assertThat(client.getUpdateCount()).isGreaterThanOrEqualTo(6);
- assertEquals(2, client.getErrorCount());
- assertEquals(6, clientCRLStore.getLatestCrlId());
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- client.stop(true);
- server.stop();
- }
- }
-}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/ListSubcommand.java
index ea08983814..e504d75b47 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/ListSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/ListSubcommand.java
@@ -68,8 +68,9 @@ public class ListSubcommand extends ScmCertSubcommand {
private String role;
@Option(names = {"-t", "--type"},
- description = "Filter certificate by the type: valid or revoked",
- defaultValue = "valid", showDefaultValue = Visibility.ALWAYS)
+ description = "This option is unused currently, and has no effect on the output.",
+ defaultValue = "VALID", showDefaultValue = Visibility.NEVER)
+ @Deprecated
private String type;
@Option(names = { "--json" },
@@ -89,10 +90,8 @@ public class ListSubcommand extends ScmCertSubcommand {
@Override
protected void execute(SCMSecurityProtocol client) throws IOException {
- boolean isRevoked = type.equalsIgnoreCase("revoked");
HddsProtos.NodeType nodeType = parseCertRole(role);
- List<String> certPemList = client.listCertificate(nodeType,
- startSerialId, count, isRevoked);
+ List<String> certPemList = client.listCertificate(nodeType, startSerialId, count);
if (count == certPemList.size()) {
err.println("The certificate list could be longer than the batch size: "
+ count + ". Please use the \"-c\" option to see more" +
@@ -100,8 +99,7 @@ public class ListSubcommand extends ScmCertSubcommand {
}
if (json) {
- err.println("Certificate list:(Type=" + type.toUpperCase() +
- ", BatchSize=" + count + ", CertCount=" + certPemList.size() + ")");
+ err.println("Certificate list:(BatchSize=" + count + ", CertCount=" + certPemList.size() + ")");
List<Certificate> certList = new ArrayList<>();
for (String certPemStr : certPemList) {
try {
@@ -117,8 +115,7 @@ public class ListSubcommand extends ScmCertSubcommand {
return;
}
- System.out.printf("Certificate list:(Type=%s, BatchSize=%s, CertCount=%s)%n",
- type.toUpperCase(), count, certPemList.size());
+ System.out.printf("Certificate list:(BatchSize=%s, CertCount=%s)%n", count, certPemList.size());
printCertList(certPemList);
}
diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/cert.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/cert.robot
index 818fbc115b..43aac21fae 100644
--- a/hadoop-ozone/dist/src/main/smoketest/admincli/cert.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/admincli/cert.robot
@@ -26,7 +26,7 @@ Setup Test
List certificates
Pass Execution If '${SECURITY_ENABLED}' == 'false' N/A
${output} = Execute ozone admin cert list
- Should Contain ${output} Certificate list:(Type=
+ Should Contain ${output} Certificate list:(BatchSize=
List certificates as JSON
Pass Execution If '${SECURITY_ENABLED}' == 'false' N/A
@@ -34,6 +34,6 @@ List certificates as JSON
${output} Execute cat outStream | jq -r '.[0] | keys'
Should Contain ${output} serialNumber
${errOutput} = Execute cat errStream
- Should Contain ${errOutput} Certificate list:(Type=
+ Should Contain ${errOutput} Certificate list:(BatchSize=
Execute rm outStream
Execute rm errStream
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/admin-cert.robot b/hadoop-ozone/dist/src/main/smoketest/security/admin-cert.robot
index 524345dd3e..adc1c195c9 100644
--- a/hadoop-ozone/dist/src/main/smoketest/security/admin-cert.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/security/admin-cert.robot
@@ -32,11 +32,12 @@ Setup Test
*** Test Cases ***
List valid certificates
${output} = Execute ozone admin cert list
- Should Contain ${output} Type=VALID
+ Should Contain ${output} Certificate list:(BatchSize=20,
+# Currently until we have revocation support, the output is expected to be the same as the simple list without -t.
List revoked certificates
${output} = Execute ozone admin cert list -t revoked
- Should Contain ${output} Certificate list:(Type=REVOKED, BatchSize=20, CertCount=0)
+ Should Contain ${output} Certificate list:(BatchSize=20,
Info of the cert
${output} = Execute for id in $(ozone admin cert list -c 1|grep UTC|awk '{print $1}'); do ozone admin cert info $id; done
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 16ef88177f..39c2250b73 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -661,7 +661,6 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
conf.set(scmHttpAddrKey, localhostWithFreePort());
conf.set(scmHttpsAddrKey, localhostWithFreePort());
conf.set(scmSecurityAddrKey, localhostWithFreePort());
- conf.set("ozone.scm.update.service.port", "0");
int ratisPort = getFreePort();
conf.setInt(scmRatisPortKey, ratisPort);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java
index 77970ad447..099c1d2e1f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
@@ -252,7 +251,6 @@ public final class TestDelegationToken {
}
private void initSCM() throws IOException {
- DefaultCAServer.setTestSecureFlag(true);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 8087d1f0e4..cc1f93fbc1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -84,7 +84,6 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare.addAll(Arrays.asList(
HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA,
HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT,
- HddsConfigKeys.HDDS_X509_CRL_NAME, // HDDS-2873
HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED,
OMConfigKeys.OZONE_OM_NODES_KEY,
OMConfigKeys.OZONE_OM_DECOMMISSIONED_NODES_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 26ecb34c88..f099fd3a92 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -69,7 +69,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.Defaul
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
@@ -473,7 +472,6 @@ final class TestSecureOzoneCluster {
Files.createDirectories(scmPath);
conf.set(OZONE_METADATA_DIRS, scmPath.toString());
- DefaultCAServer.setTestSecureFlag(true);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org