You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/04/07 18:13:15 UTC
[ozone] branch HDDS-3698-nonrolling-upgrade updated (a55f657 ->
baeb46d)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from a55f657 Merge remote-tracking branch 'upstream/master' into ssh-upstream-upgrade-branch
add 68265b8 HDDS-4925. Introduce ContainerBalancer in SCM with start/stop capabilities. (#2097)
add ee4f759 HDDS-5011. Introduce Java based ReplicationConfig implementation (#2089)
add 33ddcb3 HDDS-5062. Add a config to bypass clusterId validation for bootstrapping SCM. (#2114)
add 6822b10 HDDS-5032. Fix findbugs (#2120)
add e40c638 HDDS-5027. [SCM HA Security] Handle leader changes during bootstrap. (#2113)
add 534eefa HDDS-5056. Avoid false positiver error messages during pipeline creations (#2105)
new baeb46d Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hdds/client/RatisReplicationConfig.java | 82 ++++++++++++
.../hadoop/hdds/client/ReplicationConfig.java | 98 +++++++++++++++
.../hdds/client/StandaloneReplicationConfig.java} | 47 ++++---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 +++
.../common/src/main/resources/ozone-default.xml | 9 ++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 1 +
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../certificate/authority/CertificateStore.java | 5 +
.../hdds/scm/protocol/TestReplicationConfig.java | 66 ++++++++++
.../scm/container/balancer/ContainerBalancer.java | 137 +++++++++++++++++++++
.../balancer/ContainerBalancerConfiguration.java | 115 +++++++++++++++++
.../hdds/scm/container/balancer}/package-info.java | 6 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 94 ++++++++++++--
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 6 +
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 25 +++-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 4 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 17 ++-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 12 +-
.../hdds/scm/server/StorageContainerManager.java | 107 +++++++++-------
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 19 +++
22 files changed, 783 insertions(+), 96 deletions(-)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheKey.java => common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java} (51%)
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/ha => server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer}/package-info.java (86%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'ssh-upstream/master'
into ssh-upstream-upgrade-branch
Posted by av...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit baeb46df3f799508100690a501c4af93d7281c58
Merge: a55f657 534eefa
Author: Aravindan Vijayan <av...@cloudera.com>
AuthorDate: Wed Apr 7 11:12:44 2021 -0700
Merge remote-tracking branch 'ssh-upstream/master' into ssh-upstream-upgrade-branch
.../hadoop/hdds/client/RatisReplicationConfig.java | 82 ++++++++++++
.../hadoop/hdds/client/ReplicationConfig.java | 98 +++++++++++++++
.../hdds/client/StandaloneReplicationConfig.java | 72 +++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 +++
.../common/src/main/resources/ozone-default.xml | 9 ++
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 1 +
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../certificate/authority/CertificateStore.java | 5 +
.../hdds/scm/protocol/TestReplicationConfig.java | 66 ++++++++++
.../scm/container/balancer/ContainerBalancer.java | 137 +++++++++++++++++++++
.../balancer/ContainerBalancerConfiguration.java | 115 +++++++++++++++++
.../hdds/scm/container/balancer/package-info.java | 21 ++++
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 94 ++++++++++++--
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 6 +
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 25 +++-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 4 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 17 ++-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 12 +-
.../hdds/scm/server/StorageContainerManager.java | 107 +++++++++-------
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 19 +++
22 files changed, 846 insertions(+), 73 deletions(-)
diff --cc hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
index 0fe3ec1,69848dc..8cf584d
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
@@@ -45,5 -45,5 +45,6 @@@ public enum ConfigTag
RECON,
DELETION,
HA,
- BALANCER
++ BALANCER,
+ UPGRADE
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index e3d4bca,76c87b3..a3252c8
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -86,7 -49,8 +50,9 @@@ import org.apache.hadoop.hdds.scm.conta
import org.apache.hadoop.hdds.scm.container.ContainerActionsHandler;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+ import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl;
+ import org.apache.hadoop.hdds.scm.container.ContainerManagerV2;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
@@@ -103,30 -79,31 +81,36 @@@ import org.apache.hadoop.hdds.scm.net.N
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.node.DeadNodeHandler;
import org.apache.hadoop.hdds.scm.node.NewNodeHandler;
-import org.apache.hadoop.hdds.scm.node.NodeDecommissionManager;
+import org.apache.hadoop.hdds.scm.node.StartDatanodeAdminHandler;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeReportHandler;
-import org.apache.hadoop.hdds.scm.node.NonHealthyToHealthyNodeHandler;
+import org.apache.hadoop.hdds.scm.node.NonHealthyToReadOnlyHealthyNodeHandler;
+import org.apache.hadoop.hdds.scm.node.ReadOnlyHealthyToHealthyNodeHandler;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
-import org.apache.hadoop.hdds.scm.node.StartDatanodeAdminHandler;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.node.NodeDecommissionManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
- import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl;
+ import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
+import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizer;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
+ import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
+ import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile;
+ import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
+ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+ import org.apache.hadoop.hdds.utils.HAUtils;
+ import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.IOUtils;
@@@ -149,10 -124,25 +133,26 @@@ import org.apache.hadoop.util.JvmPauseM
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+ import javax.management.ObjectName;
+ import java.io.IOException;
+ import java.math.BigInteger;
+ import java.net.InetAddress;
+ import java.net.InetSocketAddress;
+ import java.security.cert.CertificateException;
+ import java.security.cert.X509Certificate;
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.UUID;
+ import java.util.concurrent.ConcurrentMap;
+ import java.util.concurrent.TimeUnit;
+
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
+import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED;
import static org.apache.hadoop.hdds.utils.HAUtils.checkSecurityAndSCMHAEnabled;
+import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA_PREFIX;
@@@ -245,10 -235,10 +245,12 @@@ public final class StorageContainerMana
private NetworkTopology clusterMap;
private PipelineChoosePolicy pipelineChoosePolicy;
+ private HDDSLayoutVersionManager scmLayoutVersionManager;
+ private UpgradeFinalizer<StorageContainerManager> upgradeFinalizer;
private final SCMHANodeDetails scmHANodeDetails;
+ private ContainerBalancer containerBalancer;
+
/**
* Creates a new StorageContainerManager. Configuration will be
* updated with information on the actual listening addresses used
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org