You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/04/07 18:13:16 UTC
[ozone] 01/01: Merge remote-tracking branch 'ssh-upstream/master'
into ssh-upstream-upgrade-branch
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit baeb46df3f799508100690a501c4af93d7281c58
Merge: a55f657 534eefa
Author: Aravindan Vijayan <av...@cloudera.com>
AuthorDate: Wed Apr 7 11:12:44 2021 -0700
Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
.../hadoop/hdds/client/RatisReplicationConfig.java | 82 ++++++++++++
.../hadoop/hdds/client/ReplicationConfig.java | 98 +++++++++++++++
.../hdds/client/StandaloneReplicationConfig.java | 72 +++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 +++
.../common/src/main/resources/ozone-default.xml | 9 ++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 1 +
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../certificate/authority/CertificateStore.java | 5 +
.../hdds/scm/protocol/TestReplicationConfig.java | 66 ++++++++++
.../scm/container/balancer/ContainerBalancer.java | 137 +++++++++++++++++++++
.../balancer/ContainerBalancerConfiguration.java | 115 +++++++++++++++++
.../hdds/scm/container/balancer/package-info.java | 21 ++++
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 94 ++++++++++++--
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 6 +
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 25 +++-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 4 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 17 ++-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 12 +-
.../hdds/scm/server/StorageContainerManager.java | 107 +++++++++-------
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 19 +++
22 files changed, 846 insertions(+), 73 deletions(-)
diff --cc hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 0fe3ec1,69848dc..8cf584d
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@@ -45,5 -45,5 +45,6 @@@ public enum ConfigTag
RECON,
DELETION,
HA,
- BALANCER
++ BALANCER,
+ UPGRADE
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index e3d4bca,76c87b3..a3252c8
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -86,7 -49,8 +50,9 @@@ import org.apache.hadoop.hdds.scm.conta
import org.apache.hadoop.hdds.scm.container.ContainerActionsHandler;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+ import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl;
+ import org.apache.hadoop.hdds.scm.container.ContainerManagerV2;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
@@@ -103,30 -79,31 +81,36 @@@ import org.apache.hadoop.hdds.scm.net.N
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.node.DeadNodeHandler;
import org.apache.hadoop.hdds.scm.node.NewNodeHandler;
-import org.apache.hadoop.hdds.scm.node.NodeDecommissionManager;
+import org.apache.hadoop.hdds.scm.node.StartDatanodeAdminHandler;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeReportHandler;
-import org.apache.hadoop.hdds.scm.node.NonHealthyToHealthyNodeHandler;
+import org.apache.hadoop.hdds.scm.node.NonHealthyToReadOnlyHealthyNodeHandler;
+import org.apache.hadoop.hdds.scm.node.ReadOnlyHealthyToHealthyNodeHandler;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
-import org.apache.hadoop.hdds.scm.node.StartDatanodeAdminHandler;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.node.NodeDecommissionManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
- import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl;
+ import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
+import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizer;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
+ import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
+ import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile;
+ import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
+ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+ import org.apache.hadoop.hdds.utils.HAUtils;
+ import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.IOUtils;
@@@ -149,10 -124,25 +133,26 @@@ import org.apache.hadoop.util.JvmPauseM
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+ import javax.management.ObjectName;
+ import java.io.IOException;
+ import java.math.BigInteger;
+ import java.net.InetAddress;
+ import java.net.InetSocketAddress;
+ import java.security.cert.CertificateException;
+ import java.security.cert.X509Certificate;
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.UUID;
+ import java.util.concurrent.ConcurrentMap;
+ import java.util.concurrent.TimeUnit;
+
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
+import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED;
import static org.apache.hadoop.hdds.utils.HAUtils.checkSecurityAndSCMHAEnabled;
+import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA_PREFIX;
@@@ -245,10 -235,10 +245,12 @@@ public final class StorageContainerMana
private NetworkTopology clusterMap;
private PipelineChoosePolicy pipelineChoosePolicy;
+ private HDDSLayoutVersionManager scmLayoutVersionManager;
+ private UpgradeFinalizer<StorageContainerManager> upgradeFinalizer;
private final SCMHANodeDetails scmHANodeDetails;
+ private ContainerBalancer containerBalancer;
+
/**
* Creates a new StorageContainerManager. Configuration will be
* updated with information on the actual listening addresses used
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org