You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/03/11 20:29:57 UTC
[ozone] 01/01: Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade-merge-candidate
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 7eefee7c08a5c410072d7ce11e5a54e8ae01fe6d
Merge: df57ebc f8f1b5f
Author: Ethan Rose <er...@apache.org>
AuthorDate: Thu Mar 11 15:14:41 2021 -0500
Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-merge-candidate
* upstream/master: (29 commits)
HDDS-4741. Modularize upgrade test (#1928)
HDDS-4864. Add acceptance tests to certify Ozone with boto3 python client. (#1976)
HDDS-4791. StateContext.getReports may return list with size larger t… (#1892)
HDDS-4867. Ozone admin datanode list should report dead and stale nodes (#1966)
HDDS-4858. Useless Maven cache cleanup (#1956)
HDDS-4769. Simplify insert operation of ContainerAttribute (#1865)
HDDS-4847. Fix typo in name of IdentityService (#1941)
HDDS-4869. Bump jackson version number (#1963)
HDDS-4871. Fix intellij runConfigurations for datanode (#1968)
HDDS-4870. Bump jetty version (#1964)
HDDS-4722. Creating RDBStore fails due to RDBMetrics instance race (#1820)
HDDS-4138. Improve crc efficiency by using Java.util.zip.CRC when available (#1950)
HDDS-4816. Add UsageInfoSubcommand to get Datanode usage information. (#1919)
HDDS-4754. Make scm heartbeat rpc retry interval configurable (#1942)
HDDS-4832. Show Datanode OperationalState in Recon (#1937)
HDDS-4653. Support TDE for MPU Keys on Encrypted Buckets (#1766)
HDDS-4853. libexec/entrypoint.sh might copy from wrong path (#1951)
HDDS-4857. Format ReplicationType.java which indentation are confusion (#1952)
HDDS-4850. Intermittent failure in ozonesecure due to unable to allocate block (#1948)
HDDS-4808. Add Genesis benchmark for various CRC implementations (#1910)
...
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
hadoop-hdds/interface-client/src/main/proto/hdds.proto
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
.github/workflows/post-commit.yml | 18 +-
.../java/org/apache/hadoop/hdds/JavaUtils.java | 44 +++
.../apache/hadoop/hdds/client/ReplicationType.java | 4 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 3 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 15 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 1 +
.../hadoop/hdds/scm/pipeline/PipelineID.java | 2 +
.../protocol/StorageContainerLocationProtocol.java | 21 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../org/apache/hadoop/ozone/common/Checksum.java | 6 +-
.../ozone/common/ChecksumByteBufferFactory.java | 93 +++++
.../ozone/common/ChecksumByteBufferImpl.java | 66 ++++
.../hadoop/ozone/common/NativeCheckSumCRC32.java | 85 +++++
.../ozone/common/PureJavaCrc32ByteBuffer.java | 3 +-
.../ozone/common/PureJavaCrc32CByteBuffer.java | 3 +-
.../org/apache/hadoop/util/NativeCRC32Wrapper.java | 72 ++++
.../java/org/apache/hadoop/util/package-info.java | 14 +-
.../common/src/main/resources/ozone-default.xml | 12 +-
.../ozone/common/TestChecksumByteBuffer.java | 4 +-
.../common/TestChecksumImplsComputeSameValues.java | 99 ++++++
.../common/statemachine/DatanodeStateMachine.java | 3 +-
.../common/statemachine/SCMConnectionManager.java | 9 +-
.../common/statemachine/StateContext.java | 39 ++-
.../replication/GrpcReplicationClient.java | 18 +-
.../replication/SimpleContainerDownloader.java | 10 +-
.../common/statemachine/TestStateContext.java | 36 ++
hadoop-hdds/docs/content/recipe/BotoClient.md | 189 +++++++++++
...inerLocationProtocolClientSideTranslatorPB.java | 59 +++-
.../x509/certificate/authority/CRLApprover.java | 32 +-
.../certificate/authority/CertificateServer.java | 31 +-
.../certificate/authority/CertificateStore.java | 34 +-
.../certificate/authority/DefaultCAServer.java | 39 ++-
.../certificate/authority/DefaultCRLApprover.java | 58 ++++
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 168 +++++++++
.../hdds/security/x509/crl/package-info.java | 20 +-
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 95 ++++++
.../hadoop/hdds/utils/DBCheckpointServlet.java | 113 +++----
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 14 +
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +-
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +-
.../x509/certificate/authority/MockCAStore.java | 14 +-
.../certificate/authority/TestDefaultCAServer.java | 68 +++-
.../src/main/proto/ScmAdminProtocol.proto | 25 +-
.../interface-client/src/main/proto/hdds.proto | 14 +
.../container/placement/metrics/SCMMetrics.java | 8 +
.../scm/container/states/ContainerAttribute.java | 29 +-
.../hadoop/hdds/scm/metadata/CRLInfoCodec.java | 58 ++++
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 23 +-
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 16 +
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 32 ++
.../hdds/scm/node/NodeDecommissionManager.java | 29 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 18 +-
...inerLocationProtocolServerSideTranslatorPB.java | 71 ++--
.../hadoop/hdds/scm/server/SCMCertStore.java | 101 ++++--
.../hdds/scm/server/SCMClientProtocolServer.java | 105 +++++-
.../hdds/scm/server/SCMDBCheckpointServlet.java | 64 ++++
.../hdds/scm/server/StorageContainerManager.java | 46 ++-
.../server/StorageContainerManagerHttpServer.java | 8 +-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +-
.../container/states/TestContainerAttribute.java | 2 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 11 +
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 269 +++++++++++++++
.../hdds/scm/cli/ContainerOperationClient.java | 36 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 39 ++-
.../hdds/scm/cli/datanode/DatanodeCommands.java | 8 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 14 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 8 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 16 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 17 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 103 ++++++
.../scm/cli/datanode/TestListInfoSubcommand.java | 31 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 43 +++
.../client/io/MultipartCryptoKeyInputStream.java | 375 +++++++++++++++++++++
.../ozone/client/io/OzoneCryptoInputStream.java | 57 ++++
.../hadoop/ozone/client/io/OzoneInputStream.java | 3 +
.../hadoop/ozone/client/io/OzoneOutputStream.java | 7 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 73 ++--
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 7 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 35 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 29 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 6 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 2 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 2 +-
...{IdentitiyService.java => IdentityService.java} | 2 +-
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone-mr/test.sh | 2 +-
.../src/main/compose/ozonesecure/docker-config | 5 +
.../dist/src/main/compose/ozonesecure/test.sh | 7 +-
hadoop-ozone/dist/src/main/compose/restart/test.sh | 13 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 71 +++-
.../{versions/ozone-0.5.0.sh => 0.5.0/test.sh} | 9 +-
.../{versions/ozone-1.0.0.sh => 1.0.0/test.sh} | 9 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 72 ++--
.../dist/src/main/compose/upgrade/testlib.sh | 80 +++++
.../test.sh => upgrade/upgrade_to_current.sh} | 51 ++-
.../test.sh => upgrade/upgrade_to_release.sh} | 53 ++-
.../src/main/compose/upgrade/versions/README.md | 15 -
.../versions/ozone-0.5.0.sh => versions/0.5.0.sh} | 15 +-
.../versions/ozone-0.5.0.sh => versions/1.0.0.sh} | 15 +-
.../versions/ozone-0.5.0.sh => versions/1.1.0.sh} | 15 +-
.../dist/src/main/compose/versions/README.md | 23 ++
.../dist/src/main/compose/xcompat/clients.yaml | 9 -
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 11 +-
.../dist/src/main/dockerlibexec/entrypoint.sh | 2 +-
.../dist/src/main/k8s/examples/test-all.sh | 2 +-
.../smoketest/admincli/replicationmanager.robot | 10 +-
.../src/main/smoketest/compatibility/read.robot | 11 +-
.../src/main/smoketest/compatibility/write.robot | 10 +-
hadoop-ozone/dist/src/main/smoketest/csi.robot | 2 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 8 +-
.../write.robot => replication/wait.robot} | 17 +-
.../main/smoketest/{csi.robot => s3/boto3.robot} | 24 +-
.../dist/src/main/smoketest/s3/boto_client.py | 264 +++++++++++++++
.../dist/src/shell/upgrade/{1.0.0.sh => v1.sh} | 2 +-
.../upgrade/{1.0.0 => v1}/01-migrate-scm-db.sh | 0
.../scm/TestSCMDbCheckpointServlet.java} | 144 ++------
.../client/rpc/TestOzoneAtRestEncryption.java | 198 ++++++++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 17 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 54 +--
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 +
.../src/main/proto/OmClientProtocol.proto | 3 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 226 +------------
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 48 +--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 6 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 31 ++
.../S3InitiateMultipartUploadRequest.java | 18 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 23 +-
.../protocolPB/OzoneManagerRequestHandler.java | 14 +-
.../TestS3InitiateMultipartUploadRequest.java | 60 ----
.../hadoop/ozone/recon/api/NodeEndpoint.java | 3 +
.../ozone/recon/api/types/DatanodeMetadata.java | 15 +
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 15 +-
.../webapps/recon/ozone-recon-web/api/db.json | 286 +++++++++++++++-
.../ozone-recon-web/src/types/datanode.types.tsx | 12 +-
.../src/views/datanodes/datanodes.tsx | 59 +++-
.../hadoop/ozone/recon/api/TestEndpoints.java | 47 +++
.../ozone/recon/scm/TestReconNodeManager.java | 34 +-
.../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 138 ++++++++
.../ozone/genesis/BenchMarkCRCStreaming.java | 170 ++++++++++
.../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +-
pom.xml | 4 +-
tools/fault-injection-service/CMakeLists.txt | 229 ++++++-------
tools/fault-injection-service/README.md | 37 +-
152 files changed, 4943 insertions(+), 1278 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index bab99b4,58aa0e6..5a7e809
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@@ -284,10 -283,16 +284,21 @@@ public interface ScmClient extends Clos
*/
boolean getReplicationManagerStatus() throws IOException;
+ StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
+ throws IOException;
+
+ StatusAndMessages queryUpgradeFinalizationProgress(String upgradeClientID,
- boolean force)
- throws IOException;
++ boolean force) throws IOException;
++
+ /**
+ * Get usage information of datanode by ipaddress or uuid.
+ *
+ * @param ipaddress datanode ipaddress String
+ * @param uuid datanode uuid String
+ * @return List of DatanodeUsageInfo. Each element contains info such as
+ * capacity, SCMused, and remaining space.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(String ipaddress,
- String uuid)
- throws IOException;
++ String uuid) throws IOException;
}
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 2cb7fe1,20e5f6d..46cf170
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@@ -30,9 -31,6 +31,8 @@@ import java.io.IOException
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
- import org.apache.hadoop.security.KerberosInfo;
+
/**
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
@@@ -242,10 -240,16 +242,21 @@@ public interface StorageContainerLocati
*/
boolean getReplicationManagerStatus() throws IOException;
+ StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
+ throws IOException;
+
+ StatusAndMessages queryUpgradeFinalizationProgress(String upgradeClientID,
- boolean force)
- throws IOException;
++ boolean force) throws IOException;
++
+ /**
+ * Get Datanode usage information by ip or uuid.
+ *
+ * @param ipaddress - datanode IP address String
+ * @param uuid - datanode UUID String
+ * @return List of DatanodeUsageInfo. Each element contains info such as
+ * capacity, SCMused, and remaining space.
+ * @throws IOException
+ */
+ List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(String ipaddress,
- String uuid)
- throws IOException;
++ String uuid) throws IOException;
}
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index fdbb3a0,0651707..5ac27f5
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -397,6 -393,9 +399,8 @@@ public final class OzoneConsts
public static final String OM_RATIS_SNAPSHOT_DIR = "snapshot";
- public static final long DEFAULT_OM_UPDATE_ID = -1L;
-
+ public static final long DEFAULT_OM_UPDATE_ID = -1L;
+ // CRL Sequence Id
+ public static final String CRL_SEQUENCE_ID_KEY = "CRL_SEQUENCE_ID";
}
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 6aaa373,f46e6e2..a2930c8
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@@ -45,10 -35,12 +43,10 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
- import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineBatchRequestProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
@@@ -80,12 -72,14 +77,16 @@@ import org.apache.hadoop.hdds.tracing.T
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
- import com.google.common.base.Preconditions;
- import com.google.protobuf.RpcController;
- import com.google.protobuf.ServiceException;
+ import java.io.Closeable;
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.function.Consumer;
import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION;
@@@ -617,47 -611,33 +618,73 @@@ public final class StorageContainerLoca
}
+ /**
+ * Builds request for datanode usage information and receives response.
+ *
+ * @param ipaddress - Address String
+ * @param uuid - UUID String
+ * @return List of DatanodeUsageInfo. Each element contains info such as
+ * capacity, SCMUsed, and remaining space.
+ * @throws IOException
+ */
+ @Override
+ public List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(
+ String ipaddress, String uuid) throws IOException {
+
+ DatanodeUsageInfoRequestProto request =
+ DatanodeUsageInfoRequestProto.newBuilder()
+ .setIpaddress(ipaddress)
+ .setUuid(uuid)
+ .build();
+
+ DatanodeUsageInfoResponseProto response =
+ submitRequest(Type.DatanodeUsageInfo,
+ builder -> builder.setDatanodeUsageInfoRequest(request))
+ .getDatanodeUsageInfoResponse();
+ return response.getInfoList();
+ }
+
@Override
+ public StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
+ throws IOException {
+ FinalizeScmUpgradeRequestProto req = FinalizeScmUpgradeRequestProto.
+ newBuilder()
+ .setUpgradeClientId(upgradeClientID)
+ .build();
+
+ FinalizeScmUpgradeResponseProto response =
+ submitRequest(Type.FinalizeScmUpgrade,
+ builder -> builder.setFinalizeScmUpgradeRequest(req))
+ .getFinalizeScmUpgradeResponse();
+
+ UpgradeFinalizationStatus status = response.getStatus();
+ return new StatusAndMessages(
+ UpgradeFinalizer.Status.valueOf(status.getStatus().name()),
+ status.getMessagesList());
+ }
+
+ @Override
+ public StatusAndMessages queryUpgradeFinalizationProgress(
+ String upgradeClientID, boolean force) throws IOException {
+ QueryUpgradeFinalizationProgressRequestProto req =
+ QueryUpgradeFinalizationProgressRequestProto.
+ newBuilder()
+ .setUpgradeClientId(upgradeClientID)
+ .setTakeover(force)
+ .build();
+
+ QueryUpgradeFinalizationProgressResponseProto response =
+ submitRequest(Type.QueryUpgradeFinalizationProgress,
+ builder -> builder.setQueryUpgradeFinalizationProgressRequest(req))
+ .getQueryUpgradeFinalizationProgressResponse();
+
+ UpgradeFinalizationStatus status = response.getStatus();
+ return new StatusAndMessages(
+ UpgradeFinalizer.Status.valueOf(status.getStatus().name()),
+ status.getMessagesList());
+ }
+
+ @Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
}
diff --cc hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index dde514e,3be47ea..5203c21
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@@ -66,9 -66,7 +66,10 @@@ message ScmContainerLocationRequest
optional DecommissionNodesRequestProto decommissionNodesRequest = 27;
optional RecommissionNodesRequestProto recommissionNodesRequest = 28;
optional StartMaintenanceNodesRequestProto startMaintenanceNodesRequest = 29;
- optional FinalizeScmUpgradeRequestProto finalizeScmUpgradeRequest = 30;
+ optional DatanodeUsageInfoRequestProto DatanodeUsageInfoRequest = 30;
++ optional FinalizeScmUpgradeRequestProto finalizeScmUpgradeRequest = 31;
+ optional QueryUpgradeFinalizationProgressRequestProto
- queryUpgradeFinalizationProgressRequest = 31;
++ queryUpgradeFinalizationProgressRequest = 32;
}
message ScmContainerLocationResponse {
@@@ -106,9 -104,7 +107,10 @@@
optional DecommissionNodesResponseProto decommissionNodesResponse = 27;
optional RecommissionNodesResponseProto recommissionNodesResponse = 28;
optional StartMaintenanceNodesResponseProto startMaintenanceNodesResponse = 29;
- optional FinalizeScmUpgradeResponseProto finalizeScmUpgradeResponse = 30;
+ optional DatanodeUsageInfoResponseProto DatanodeUsageInfoResponse = 30;
++ optional FinalizeScmUpgradeResponseProto finalizeScmUpgradeResponse = 31;
+ optional QueryUpgradeFinalizationProgressResponseProto
- queryUpgradeFinalizationProgressResponse = 31;
++ queryUpgradeFinalizationProgressResponse = 32;
enum Status {
OK = 1;
CONTAINER_ALREADY_EXISTS = 2;
@@@ -142,8 -138,7 +144,9 @@@ enum Type
DecommissionNodes = 22;
RecommissionNodes = 23;
StartMaintenanceNodes = 24;
- DatanodeUsageInfo = 25;
+ FinalizeScmUpgrade = 25;
+ QueryUpgradeFinalizationProgress = 26;
++ DatanodeUsageInfo = 27;
}
/**
diff --cc hadoop-hdds/interface-client/src/main/proto/hdds.proto
index b1cf85d,8250449..e8beead
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@@ -287,15 -292,10 +293,23 @@@ message BlockID
optional uint64 blockCommitSequenceId = 2 [default = 0];
}
+message UpgradeFinalizationStatus {
+ enum Status {
+ ALREADY_FINALIZED = 1;
+ STARTING_FINALIZATION = 2;
+ FINALIZATION_IN_PROGRESS = 3;
+ FINALIZATION_DONE = 4;
+ FINALIZATION_REQUIRED = 5;
+ }
+ required Status status = 1;
+ repeated string messages = 2;
+}
+
+ /**
+ * Information for Certificate Revocation List.
+ */
+ message CRLInfoProto {
+ required string x509CRL = 1;
+ required uint64 creationTimestamp = 2;
+ }
++
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 0442908,bd73d36..a3b9354
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@@ -33,10 -31,11 +32,13 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ClosePipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeUsageInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DeactivatePipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeResponseProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
@@@ -53,8 -54,8 +57,10 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.QueryUpgradeFinalizationProgressRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.QueryUpgradeFinalizationProgressResponseProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ReplicationManagerStatusResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMCloseContainerRequestProto;
@@@ -86,16 -81,16 +86,17 @@@ import org.apache.hadoop.hdds.scm.pipel
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
- import com.google.protobuf.ProtocolMessageEnum;
- import com.google.protobuf.RpcController;
- import com.google.protobuf.ServiceException;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto.Error.errorPipelineAlreadyExists;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto.Error.success;
-
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
/**
* This class is the server-side translator that forwards requests received on
@@@ -309,22 -303,14 +310,29 @@@ public final class StorageContainerLoca
.setStatus(Status.OK)
.setStartMaintenanceNodesResponse(startMaintenanceNodes(
request.getStartMaintenanceNodesRequest()))
+ .build();
+ case FinalizeScmUpgrade:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setFinalizeScmUpgradeResponse(getFinalizeScmUpgrade(
+ request.getFinalizeScmUpgradeRequest()))
+ .build();
+ case QueryUpgradeFinalizationProgress:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setQueryUpgradeFinalizationProgressResponse(
+ getQueryUpgradeFinalizationProgress(
+ request.getQueryUpgradeFinalizationProgressRequest()))
.build();
+ case DatanodeUsageInfo:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setDatanodeUsageInfoResponse(getDatanodeUsageInfo(
+ request.getDatanodeUsageInfoRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 53fab59,e1334e7..aeceb8e
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@@ -87,10 -91,6 +91,8 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+
/**
* The RPC server that listens to requests from clients.
*/
@@@ -620,19 -620,74 +622,86 @@@ public class SCMClientProtocolServer im
return scm.getReplicationManager().isRunning();
}
+ @Override
+ public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws
+ IOException {
+ return scm.finalizeUpgrade(upgradeClientID);
+ }
+
+ @Override
+ public StatusAndMessages queryUpgradeFinalizationProgress(
+ String upgradeClientID, boolean force) throws IOException {
+ return scm.queryUpgradeFinalizationProgress(upgradeClientID, force);
+ }
+
/**
+ * Get Datanode usage info (such as capacity, used) by ip or uuid.
+ *
+ * @param ipaddress - Datanode Address String
+ * @param uuid - Datanode UUID String
+ * @return List of DatanodeUsageInfo. Each element contains usage info such
+ * as capacity, SCMUsed, and remaining space.
+ * @throws IOException
+ */
+ @Override
+ public List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(
+ String ipaddress, String uuid) throws IOException {
+
+ // check admin authorisation
+ String remoteUser = getRpcRemoteUsername();
+ try {
+ getScm().checkAdminAccess(remoteUser);
+ } catch (IOException e) {
+ LOG.error("Authorisation failed", e);
+ throw e;
+ }
+
+ // get datanodes by ip or uuid
+ List<DatanodeDetails> nodes = new ArrayList<>();
+ if (!Strings.isNullOrEmpty(uuid)) {
+ nodes.add(scm.getScmNodeManager().getNodeByUuid(uuid));
+ } else if (!Strings.isNullOrEmpty(ipaddress)) {
+ nodes = scm.getScmNodeManager().getNodesByAddress(ipaddress);
+ } else {
+ throw new IOException(
+ "Could not get datanode with the specified parameters."
+ );
+ }
+
+ // get datanode usage info
+ List<HddsProtos.DatanodeUsageInfo> infoList = new ArrayList<>();
+ for (DatanodeDetails node : nodes) {
+ infoList.add(getUsageInfoFromDatanodeDetails(node));
+ }
+
+ return infoList;
+ }
+
+ /**
+ * Get usage details for a specific DatanodeDetails node.
+ *
+ * @param node - DatanodeDetails
+ * @return Usage info such as capacity, SCMUsed, and remaining space.
+ * @throws IOException
+ */
+ private HddsProtos.DatanodeUsageInfo getUsageInfoFromDatanodeDetails(
+ DatanodeDetails node) throws IOException {
+ SCMNodeStat stat = scm.getScmNodeManager().getNodeStat(node).get();
+
+ long capacity = stat.getCapacity().get();
+ long used = stat.getScmUsed().get();
+ long remaining = stat.getRemaining().get();
+
+ HddsProtos.DatanodeUsageInfo info = HddsProtos.DatanodeUsageInfo
+ .newBuilder()
+ .setCapacity(capacity)
+ .setUsed(used)
+ .setRemaining(remaining)
+ .build();
+ return info;
+ }
+
+ /**
* Queries a list of Node that match a set of statuses.
*
* <p>For example, if the nodeStatuses is HEALTHY and RAFT_MEMBER, then
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 75e6a8a,bca98ed..9c94f2b
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -126,14 -121,10 +130,11 @@@ import com.google.common.cache.Cache
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.protobuf.BlockingService;
- import org.apache.commons.lang3.tuple.Pair;
+
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
-
- import org.apache.ratis.grpc.GrpcTlsConfig;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+ import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY;
/**
* StorageContainerManager is the main entry point for the service that
diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 70c7606,baac26d..e4d7d31
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@@ -51,10 -46,16 +46,17 @@@ import org.apache.hadoop.ipc.ProtobufRp
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.UserGroupInformation;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import javax.net.SocketFactory;
+ import java.io.IOException;
+ import java.net.InetSocketAddress;
+ import java.util.List;
+ import java.util.Map;
- import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
@@@ -535,16 -532,20 +533,32 @@@ public class ContainerOperationClient i
return storageContainerLocationClient.getReplicationManagerStatus();
}
+ @Override
+ public StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
+ throws IOException {
+ return storageContainerLocationClient.finalizeScmUpgrade(upgradeClientID);
+ }
+
+ @Override
+ public StatusAndMessages queryUpgradeFinalizationProgress(
+ String upgradeClientID, boolean force) throws IOException {
+ return storageContainerLocationClient.queryUpgradeFinalizationProgress(
+ upgradeClientID, force);
+ }
++
+ /**
+ * Get Datanode Usage information by ipaddress or uuid.
+ *
+ * @param ipaddress - datanode ipaddress String
+ * @param uuid - datanode uuid String
+ * @return List of DatanodeUsageInfo. Each element contains info such as
+ * capacity, SCMused, and remaining space.
+ * @throws IOException
+ */
+ @Override
+ public List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(
+ String ipaddress, String uuid) throws IOException {
+ return storageContainerLocationClient.getDatanodeUsageInfo(ipaddress,
+ uuid);
+ }
-
}
diff --cc hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
index ba8d468,b6af473..cb7cf8a
--- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
+++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
@@@ -89,12 -90,22 +90,25 @@@ public class TestListInfoSubcommand
"^Operational State:\\s+DECOMMISSIONING$", Pattern.MULTILINE);
m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
+ for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
+ p = Pattern.compile(
+ "^Health State:\\s+"+state+"$", Pattern.MULTILINE);
+ m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
+ }
- // Ensure the nodes are ordered by health state HEALTHY, STALE, DEAD
- p = Pattern.compile(".+HEALTHY.+STALE.+DEAD.+", Pattern.DOTALL);
++ // Ensure the nodes are ordered by health state HEALTHY,
++ // HEALTHY_READONLY, STALE, DEAD
++ p = Pattern.compile(".+HEALTHY.+STALE.+DEAD.+HEALTHY_READONLY.+",
++ Pattern.DOTALL);
++
+ m = p.matcher(outContent.toString(DEFAULT_ENCODING));
+ assertTrue(m.find());
}
private List<HddsProtos.Node> getNodeDetails() {
List<HddsProtos.Node> nodes = new ArrayList<>();
- for (int i=0; i<2; i++) {
- for (int i=0; i<3; i++) {
++ for (int i=0; i<4; i++) {
HddsProtos.DatanodeDetailsProto.Builder dnd =
HddsProtos.DatanodeDetailsProto.newBuilder();
dnd.setHostName("host" + i);
@@@ -109,11 -120,16 +123,20 @@@
if (i == 0) {
builder.addNodeOperationalStates(
HddsProtos.NodeOperationalState.IN_SERVICE);
- } else {
+ builder.addNodeStates(HddsProtos.NodeState.STALE);
+ } else if (i == 1) {
builder.addNodeOperationalStates(
HddsProtos.NodeOperationalState.DECOMMISSIONING);
+ builder.addNodeStates(HddsProtos.NodeState.DEAD);
++ } else if (i == 2) {
++ builder.addNodeOperationalStates(
++ HddsProtos.NodeOperationalState.IN_SERVICE);
++ builder.addNodeStates(HddsProtos.NodeState.HEALTHY_READONLY);
+ } else {
+ builder.addNodeOperationalStates(
+ HddsProtos.NodeOperationalState.IN_SERVICE);
+ builder.addNodeStates(HddsProtos.NodeState.HEALTHY);
}
- builder.addNodeStates(HddsProtos.NodeState.HEALTHY);
builder.setNodeID(dnd.build());
nodes.add(builder.build());
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 0eac529,0b4b8ea..556e7e6
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -3832,53 -3799,13 +3834,62 @@@ public final class OzoneManager extend
return omVolumeArgs.build();
}
+ public OMLayoutVersionManager getVersionManager() {
+ return versionManager;
+ }
+
+ public OzoneManagerPrepareState getPrepareState() {
+ return prepareState;
+ }
+
+ private void instantiatePrepareState(boolean withNewSnapshot)
+ throws IOException {
+ // If the prepare marker file is present and its index matches the last
+ // transaction index in the OM DB, turn on the in memory flag to
+ // put the ozone manager in prepare mode, disallowing write requests.
+ // This must be done after metadataManager is instantiated
+ // and before the RPC server is started.
+ OMTransactionInfo txnInfo = metadataManager.getTransactionInfoTable()
+ .get(TRANSACTION_INFO_KEY);
+ prepareState = new OzoneManagerPrepareState(configuration);
+
+ // If we have no transaction info in the DB, then no prepare request
+ // could have been received, since the request would update the txn
+ // index in the DB.
+ if (txnInfo != null) {
+ // Only puts OM in prepare mode if a marker file matching the txn index
+ // is found.
+ PrepareStatus status =
+ prepareState.restorePrepare(txnInfo.getTransactionIndex());
+ if (status == PrepareStatus.PREPARE_COMPLETED) {
+ LOG.info("Ozone Manager {} restarted in prepare mode.",
+ getOMNodeId());
+ } else if (withNewSnapshot) {
+ OMTransactionInfo prepareInfo =
+ metadataManager.getTransactionInfoTable().get(PREPARE_MARKER_KEY);
+ if (prepareInfo != null &&
+ prepareInfo.getTransactionIndex() ==
+ txnInfo.getTransactionIndex()) {
+ LOG.info("Prepare Index matches last applied index, but the prepare" +
+ " marker file is not found. A snapshot is being installed in " +
+ " this OM. Enabling prepare gate.");
+ prepareState.finishPrepare(prepareInfo.getTransactionIndex());
+ } else {
+ prepareState.cancelPrepare();
+ }
+ }
+ } else {
+ // Make sure OM prepare state is clean before proceeding.
+ prepareState.cancelPrepare();
+ }
+ }
++
+ public int getMinMultipartUploadPartSize() {
+ return minMultipartUploadPartSize;
+ }
+
+ @VisibleForTesting
+ public void setMinMultipartUploadPartSize(int partSizeForTest) {
+ this.minMultipartUploadPartSize = partSizeForTest;
+ }
-
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
index 474a6e1,b5c9fdf..0cf3de4
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
@@@ -51,9 -51,7 +51,8 @@@ import java.util.Collections
import java.util.Map;
import java.util.UUID;
- import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.InitiateMultiPartUpload;
/**
* Handles initiate multipart upload request.
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index fd4af37,c4de2fb..60938c2
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@@ -54,11 -54,8 +54,10 @@@ import org.apache.hadoop.hdds.utils.db.
import com.google.common.base.Optional;
import org.apache.commons.codec.digest.DigestUtils;
- import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CompleteMultiPartUpload;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
index ce5b416,49e3fd3..6bf2a97
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
@@@ -19,9 -19,9 +19,10 @@@
package org.apache.hadoop.ozone.recon.api;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
index 5af96b3,218f717..9a7c75c
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
@@@ -27,18 -27,14 +27,19 @@@ import java.util.UUID
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
+ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
diff --cc hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
index c49104b,0cacd7a..9199745
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
@@@ -36,8 -35,8 +36,9 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
@@@ -116,14 -110,41 +117,43 @@@ public class TestReconNodeManager
reconNodeManager.addDatanodeCommand(datanodeDetails.getUuid(),
new ReregisterCommand());
+ // OperationalState sanity check
+ final DatanodeDetails dnDetails =
+ reconNodeManager.getNodeByUuid(datanodeDetails.getUuidString());
+ assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE,
+ dnDetails.getPersistedOpState());
+ assertEquals(dnDetails.getPersistedOpState(),
+ reconNodeManager.getNodeStatus(dnDetails)
+ .getOperationalState());
+ assertEquals(dnDetails.getPersistedOpStateExpiryEpochSec(),
+ reconNodeManager.getNodeStatus(dnDetails)
+ .getOpStateExpiryEpochSeconds());
+
// Upon processing the heartbeat, the illegal command should be filtered out
List<SCMCommand> returnedCmds =
- reconNodeManager.processHeartbeat(datanodeDetails);
+ reconNodeManager.processHeartbeat(datanodeDetails,
+ defaultLayoutVersionProto());
assertEquals(1, returnedCmds.size());
assertEquals(SCMCommandProto.Type.reregisterCommand,
returnedCmds.get(0).getType());
+ // Now feed a DECOMMISSIONED heartbeat of the same DN
+ datanodeDetails.setPersistedOpState(
+ HddsProtos.NodeOperationalState.DECOMMISSIONED);
+ datanodeDetails.setPersistedOpStateExpiryEpochSec(12345L);
- reconNodeManager.processHeartbeat(datanodeDetails);
++ reconNodeManager.processHeartbeat(datanodeDetails,
++ defaultLayoutVersionProto());
+ // Check both persistedOpState and NodeStatus#operationalState
+ assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED,
+ dnDetails.getPersistedOpState());
+ assertEquals(dnDetails.getPersistedOpState(),
+ reconNodeManager.getNodeStatus(dnDetails)
+ .getOperationalState());
+ assertEquals(12345L, dnDetails.getPersistedOpStateExpiryEpochSec());
+ assertEquals(dnDetails.getPersistedOpStateExpiryEpochSec(),
+ reconNodeManager.getNodeStatus(dnDetails)
+ .getOpStateExpiryEpochSeconds());
+
// Close the DB, and recreate the instance of Recon Node Manager.
eventQueue.close();
reconNodeManager.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org