You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ae...@apache.org on 2019/11/08 17:22:56 UTC
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-1880-Decom
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 88147d09b6a34f203596a26b0aef89c334d33894
Merge: 278699e ab7987c
Author: Anu Engineer <ae...@apache.org>
AuthorDate: Fri Nov 8 09:22:35 2019 -0800
Merge branch 'master' into HDDS-1880-Decom
BUILDING.txt | 511 ---------------------
CONTRIBUTION.md | 206 +++++++++
README.md | 98 ++++
README.txt | 7 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 88 ++--
.../hadoop/hdds/scm/XceiverClientManager.java | 38 +-
.../hdds/scm/client/ContainerOperationClient.java | 111 +++--
.../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 6 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 111 ++++-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 2 +-
.../scm/container/common/helpers/ExcludeList.java | 14 +-
.../hdds/security/token/BlockTokenVerifier.java | 114 ++---
.../security/token/OzoneBlockTokenIdentifier.java | 3 +-
.../hadoop/hdds/security/token/TokenVerifier.java | 13 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +
.../certificates/utils/CertificateSignRequest.java | 41 +-
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 4 +-
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 8 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 6 +
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 16 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 9 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 5 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 3 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 -
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 27 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 21 +
.../certificate/authority/TestDefaultCAServer.java | 1 +
.../certificate/authority/TestDefaultProfile.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 109 ++---
.../container/common/helpers/ContainerUtils.java | 2 +-
.../ozone/container/common/impl/ContainerData.java | 41 +-
.../common/impl/ContainerDataScanOrder.java | 57 +++
.../container/common/impl/ContainerDataYaml.java | 15 +-
.../ozone/container/common/impl/ContainerSet.java | 2 +
.../container/common/impl/HddsDispatcher.java | 41 +-
.../container/common/interfaces/Container.java | 4 +
.../transport/server/GrpcXceiverService.java | 16 -
.../server/ServerCredentialInterceptor.java | 74 ---
.../common/transport/server/XceiverServer.java | 89 ----
.../common/transport/server/XceiverServerGrpc.java | 18 +-
.../server/ratis/ContainerStateMachine.java | 34 +-
.../transport/server/ratis/XceiverServerRatis.java | 13 +-
.../container/keyvalue/KeyValueContainer.java | 16 +-
.../container/keyvalue/TarContainerPacker.java | 31 +-
.../container/keyvalue/helpers/ChunkUtils.java | 23 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 17 +-
.../container/ozoneimpl/ContainerController.java | 7 +
.../container/ozoneimpl/ContainerDataScanner.java | 31 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 7 +-
.../replication/GrpcReplicationClient.java | 39 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 -
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 1 -
.../ozone/container/common/SCMTestUtils.java | 22 +
.../common/impl/TestContainerDataYaml.java | 33 +-
.../container/common/impl/TestContainerSet.java | 59 ++-
.../container/common/impl/TestHddsDispatcher.java | 6 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../container/keyvalue/TestBlockManagerImpl.java | 11 +-
.../container/keyvalue/TestChunkManagerImpl.java | 113 +++--
.../container/keyvalue/TestKeyValueContainer.java | 12 +-
.../container/keyvalue/TestTarContainerPacker.java | 204 ++++++--
.../ozoneimpl/TestContainerScrubberMetrics.java | 138 +++---
hadoop-hdds/docs/content/beyond/Containers.md | 1 -
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 4 +-
hadoop-hdds/docs/content/start/OnPrem.md | 16 -
.../server/OzoneProtocolMessageDispatcher.java | 4 +-
.../hadoop/hdds/server/events/EventQueue.java | 4 +-
.../src/main/resources/webapps/datanode/dn.js | 2 -
.../hdds/scm/container/ReplicationManager.java | 19 +-
.../metrics/SCMContainerManagerMetrics.java | 3 +-
.../algorithms/SCMContainerPlacementMetrics.java | 3 +-
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 +
.../metadata/DeletedBlocksTransactionCodec.java | 5 +
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 5 +
.../hdds/scm/metadata/X509CertificateCodec.java | 5 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 3 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 +-
.../hdds/scm/server/SCMContainerMetrics.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 3 -
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 3 -
.../hadoop/hdds/scm/block/TestBlockManager.java | 27 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 -
.../hadoop/hdds/scm/container/MockNodeManager.java | 5 +-
.../container/TestCloseContainerEventHandler.java | 5 +-
.../scm/container/TestSCMContainerManager.java | 30 +-
.../TestSCMContainerPlacementRackAware.java | 61 ++-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 181 ++++++++
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 89 +---
.../apache/hadoop/ozone/client/ObjectStore.java | 2 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 13 +-
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 4 +
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 4 +
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 5 +
.../hadoop/ozone/om/codec/S3SecretValueCodec.java | 5 +
.../ozone/om/codec/TokenIdentifierCodec.java | 4 +
.../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 4 +
.../ozone/om/exceptions/NotLeaderException.java | 45 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 67 ++-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 28 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 53 ++-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 31 ++
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 18 +
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 29 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 78 ++--
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 3 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 8 +-
.../src/main/proto/OzoneManagerProtocol.proto | 6 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 ++-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 83 ++++
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 64 +++
...TestOmBucketInfo.java => TestOmPrefixInfo.java} | 42 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 87 ++++
.../checks/hadolint.sh} | 31 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 -
.../dist/dev-support/bin/dist-layout-stitching | 35 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 2 -
.../dist/src/main/compose/ozone-mr/common-config | 3 +-
.../{ozones3/test.sh => ozone-om-ha-s3/.env} | 18 +-
.../docker-compose.yaml | 10 +
.../{ozone-om-ha => ozone-om-ha-s3}/docker-config | 9 +-
.../compose/{ozones3 => ozone-om-ha-s3}/test.sh | 4 +-
.../test.sh => ozone-om-ha/.ssh/authorized_keys} | 18 +-
.../{ozones3/test.sh => ozone-om-ha/.ssh/config} | 20 +-
.../test.sh => ozone-om-ha/.ssh/environment} | 18 +-
.../dist/src/main/compose/ozone-om-ha/.ssh/id_rsa | 42 ++
.../test.sh => ozone-om-ha/.ssh/id_rsa.pub} | 18 +-
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 47 ++
.../main/compose/ozone-om-ha/docker-compose.yaml | 20 +-
.../src/main/compose/ozone-om-ha/docker-config | 59 ++-
.../{ozones3/test.sh => ozone-om-ha/startOM.sh} | 22 +-
.../{ozones3/test.sh => ozone-om-ha/stopOM.sh} | 18 +-
.../main/compose/{ozones3 => ozone-om-ha}/test.sh | 12 +-
.../src/main/compose/ozone-recon/docker-config | 3 +-
.../src/main/compose/ozone-topology/docker-config | 3 +-
.../dist/src/main/compose/ozone/docker-config | 3 +-
.../src/main/compose/ozoneblockade/docker-config | 1 -
.../dist/src/main/compose/ozoneperf/README.md | 20 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 9 -
.../dist/src/main/compose/ozoneperf/docker-config | 3 +-
.../{ozones3/test.sh => ozoneperf/freon-ockg.yaml} | 25 +-
.../{ozones3/test.sh => ozoneperf/freon-rk.yaml} | 25 +-
.../src/main/compose/ozones3-haproxy/docker-config | 3 +-
.../dist/src/main/compose/ozones3/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 2 +
.../src/main/compose/ozonescripts/docker-config | 1 -
.../src/main/compose/ozonesecure-mr/docker-config | 3 +-
.../src/main/compose/ozonesecure/docker-config | 3 +-
.../src/main/k8s/definitions/ozone/config.yaml | 1 -
.../examples/getting-started/config-configmap.yaml | 1 -
.../k8s/examples/minikube/config-configmap.yaml | 1 -
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 -
.../main/k8s/examples/ozone/config-configmap.yaml | 1 -
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +
.../dist/src/main/smoketest/omha/testOMHA.robot | 176 +++++++
.../dist/src/main/smoketest/s3/commonawslib.robot | 13 +-
.../main/smoketest/security/ozone-secure-s3.robot | 2 +-
.../conf/dn-audit-log4j2.properties | 0
.../dist/src/{main => shell}/conf/log4j.properties | 21 -
.../conf/om-audit-log4j2.properties | 0
.../conf/ozone-shell-log4j.properties | 0
.../dist/src/{main => shell}/conf/ozone-site.xml | 0
.../conf/scm-audit-log4j2.properties | 0
.../dist/src/shell/hdds}/hadoop-config.cmd | 0
.../dist/src/shell/hdds}/hadoop-config.sh | 0
.../dist/src/shell/hdds}/hadoop-daemons.sh | 0
.../dist/src/shell/hdds}/hadoop-functions.sh | 0
.../dist/src/shell/hdds}/workers.sh | 0
.../src/main/bin => dist/src/shell/ozone}/ozone | 0
.../bin => dist/src/shell/ozone}/ozone-config.sh | 0
.../bin => dist/src/shell/ozone}/start-ozone.sh | 9 -
.../bin => dist/src/shell/ozone}/stop-ozone.sh | 9 -
.../shell}/shellprofile.d/hadoop-ozone-manager.sh | 8 +-
.../src/shell}/shellprofile.d/hadoop-ozone.sh | 0
.../network-tests/src/test/compose/docker-config | 1 -
.../hadoop/ozone/insight/BaseInsightPoint.java | 56 +--
.../apache/hadoop/ozone/insight/InsightPoint.java | 6 +-
.../apache/hadoop/ozone/insight/LogSubcommand.java | 26 +-
.../ozone/insight/datanode/RatisInsight.java | 5 +
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 73 +++
.../TestContainerStateManagerIntegration.java | 134 +++---
.../metrics/TestSCMContainerManagerMetrics.java | 6 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 207 ---------
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 6 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 -
.../hadoop/ozone/TestContainerOperations.java | 20 +-
.../TestContainerStateMachineIdempotency.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 -
.../hadoop/ozone/TestStorageContainerManager.java | 11 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 9 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../ozone/client/rpc/TestKeyInputStream.java | 104 ++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 2 +-
.../ozone/client/rpc/TestWatchForCommit.java | 10 +-
.../ozone/container/ContainerTestHelper.java | 34 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 6 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 189 ++++----
.../ozone/om/TestOzoneManagerConfiguration.java | 53 ++-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 46 +-
.../ozone/ozShell/TestOzoneDatanodeShell.java | 104 +----
.../apache/hadoop/ozone/ozShell/TestS3Shell.java | 4 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 13 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 17 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 17 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 46 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 9 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 5 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 91 +++-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 31 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 9 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 2 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 5 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 36 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 32 +-
.../om/request/bucket/OMBucketCreateRequest.java | 3 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 3 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 3 +-
.../ozone/om/request/file/OMFileRequest.java | 4 +-
.../S3InitiateMultipartUploadRequest.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 23 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 6 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 6 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 2 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 6 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 6 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 6 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 8 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 69 ++-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +-
.../request/TestOMClientRequestWithUserInfo.java | 13 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 4 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../om/request/key/TestOMKeyCreateRequest.java | 4 +-
.../s3/bucket/TestS3BucketDeleteRequest.java | 3 +-
.../s3/bucket/TestS3BucketDeleteResponse.java | 3 +-
.../s3/multipart/TestS3MultipartResponse.java | 4 +-
.../security/TestOzoneBlockTokenSecretManager.java | 48 ++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 11 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 2 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 5 +
.../s3/endpoint/TestAbortMultipartUpload.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestBucketHead.java | 5 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 9 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 29 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 33 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 32 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 3 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 24 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 3 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 2 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 2 +-
.../apache/hadoop/ozone/admin/om/package-info.java | 2 +-
.../apache/hadoop/ozone/admin/package-info.java | 2 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 40 ++
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 190 ++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 8 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 8 +-
.../GenerateOzoneRequiredConfigurations.java | 4 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 11 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/genesis/BenchMarkOMClient.java | 30 +-
.../ozone/genesis/BenchMarkOzoneManager.java | 33 +-
.../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 18 +-
pom.xml | 2 +-
297 files changed, 4435 insertions(+), 2763 deletions(-)
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index 03dd829,63fe152..003035c
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@@ -29,19 -34,8 +33,15 @@@ import org.apache.hadoop.hdds.scm.net.N
import org.apache.hadoop.hdds.scm.net.NodeSchema;
import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.Collection;
- import java.util.List;
-
+ import org.apache.commons.lang.StringUtils;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
index 0000000,6f0425d..fc16e98
mode 000000,100644..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@@ -1,0 -1,180 +1,181 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hdds.scm.pipeline;
+
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.scm.TestUtils;
+ import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+ import org.apache.hadoop.hdds.scm.node.NodeManager;
++import org.apache.hadoop.hdds.scm.node.NodeStatus;
+ import org.junit.Assume;
+ import org.junit.Before;
+ import org.junit.Test;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Set;
+
+ import static org.apache.commons.collections.CollectionUtils.intersection;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
+
+ /**
+ * Test for RatisPipelineProvider.
+ */
+ public class TestRatisPipelineProvider {
+
+ private static final HddsProtos.ReplicationType REPLICATION_TYPE =
+ HddsProtos.ReplicationType.RATIS;
+
+ private NodeManager nodeManager;
+ private PipelineProvider provider;
+ private PipelineStateManager stateManager;
+
+ @Before
+ public void init() throws Exception {
+ nodeManager = new MockNodeManager(true, 10);
+ stateManager = new PipelineStateManager(new OzoneConfiguration());
+ provider = new MockRatisPipelineProvider(nodeManager,
+ stateManager, new OzoneConfiguration());
+ }
+
+ private void createPipelineAndAssertions(
+ HddsProtos.ReplicationFactor factor) throws IOException {
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline);
+
+ Pipeline pipeline1 = provider.create(factor);
+ assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE);
+ // New pipeline should not overlap with the previous created pipeline
+ assertTrue(
+ intersection(pipeline.getNodes(), pipeline1.getNodes())
+ .isEmpty());
+ stateManager.addPipeline(pipeline1);
+ }
+
+ @Test
+ public void testCreatePipelineWithFactor() throws IOException {
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline);
+
+ factor = HddsProtos.ReplicationFactor.ONE;
+ Pipeline pipeline1 = provider.create(factor);
+ assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE);
+ stateManager.addPipeline(pipeline1);
+ // New pipeline should overlap with the previous created pipeline,
+ // and one datanode should overlap between the two types.
+ assertEquals(1,
+ intersection(pipeline.getNodes(), pipeline1.getNodes()).size());
+ }
+
+ @Test
+ public void testCreatePipelineWithFactorThree() throws IOException {
+ createPipelineAndAssertions(HddsProtos.ReplicationFactor.THREE);
+ }
+
+ @Test
+ public void testCreatePipelineWithFactorOne() throws IOException {
+ createPipelineAndAssertions(HddsProtos.ReplicationFactor.ONE);
+ }
+
+ private List<DatanodeDetails> createListOfNodes(int nodeCount) {
+ List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < nodeCount; i++) {
+ nodes.add(TestUtils.randomDatanodeDetails());
+ }
+ return nodes;
+ }
+
+ @Test
+ public void testCreatePipelineWithNodes() {
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+ Pipeline pipeline =
+ provider.create(factor, createListOfNodes(factor.getNumber()));
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+
+ factor = HddsProtos.ReplicationFactor.ONE;
+ pipeline = provider.create(factor, createListOfNodes(factor.getNumber()));
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+ }
+
+ @Test
+ public void testCreatePipelinesDnExclude() throws IOException {
+ List<DatanodeDetails> healthyNodes =
- nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
++ nodeManager.getNodes(NodeStatus.inServiceHealthy());
+
+ Assume.assumeTrue(healthyNodes.size() == 8);
+
+ HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+
+ // Use up first 3 DNs for an open pipeline.
+ List<DatanodeDetails> dns = healthyNodes.subList(0, 3);
+ addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE);
+ Set<DatanodeDetails> membersOfOpenPipelines = new HashSet<>(dns);
+
+ // Use up next 3 DNs for a closed pipeline.
+ dns = healthyNodes.subList(3, 6);
+ addPipeline(dns, factor, Pipeline.PipelineState.CLOSED, REPLICATION_TYPE);
+ Set<DatanodeDetails> membersOfClosedPipelines = new HashSet<>(dns);
+
+ // only 2 healthy DNs left that are not part of any pipeline
+ Pipeline pipeline = provider.create(factor);
+ assertPipelineProperties(pipeline, factor, REPLICATION_TYPE);
+
+ List<DatanodeDetails> nodes = pipeline.getNodes();
+
+ assertTrue(
+ "nodes of new pipeline cannot be from open pipelines",
+ nodes.stream().noneMatch(membersOfOpenPipelines::contains));
+
+ assertTrue(
+ "at least 1 node should have been from members of closed pipelines",
+ nodes.stream().anyMatch(membersOfClosedPipelines::contains));
+ }
+
+ private static void assertPipelineProperties(
+ Pipeline pipeline, HddsProtos.ReplicationFactor expectedFactor,
+ HddsProtos.ReplicationType expectedReplicationType) {
+ assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState());
+ assertEquals(expectedReplicationType, pipeline.getType());
+ assertEquals(expectedFactor, pipeline.getFactor());
+ assertEquals(expectedFactor.getNumber(), pipeline.getNodes().size());
+ }
+
+ private void addPipeline(
+ List<DatanodeDetails> dns, HddsProtos.ReplicationFactor factor,
+ Pipeline.PipelineState open, HddsProtos.ReplicationType replicationType)
+ throws IOException {
+ Pipeline openPipeline = Pipeline.newBuilder()
+ .setType(replicationType)
+ .setFactor(factor)
+ .setNodes(dns)
+ .setState(open)
+ .setId(PipelineID.randomId())
+ .build();
+
+ stateManager.addPipeline(openPipeline);
+ }
+ }
diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
index 8649992,8c0fb03..75e2aa6
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
@@@ -25,12 -27,9 +27,10 @@@ import org.apache.hadoop.hdds.HddsUtils
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
- import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
- import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands;
import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands;
+import org.apache.hadoop.hdds.scm.cli.node.DatanodeAdminCommands;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
index bada595,0000000..5cf0864
mode 100644,000000..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java
@@@ -1,137 -1,0 +1,135 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static junit.framework.TestCase.assertEquals;
+import static org.apache.hadoop.hdds.HddsConfigKeys.*;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*;
+
+/**
+ * Test from the scmclient for decommission and maintenance.
+ */
+
+public class TestDecommissionAndMaintenance {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDecommissionAndMaintenance.class);
+
+ private static int numOfDatanodes = 5;
+ private MiniOzoneCluster cluster;
+
+ private ContainerOperationClient scmClient;
+
+ @Before
+ public void setUp() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ final int interval = 100;
+
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ interval, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(numOfDatanodes)
+ .build();
+ cluster.waitForClusterToBeReady();
- scmClient = new ContainerOperationClient(cluster
- .getStorageContainerLocationClient(),
- new XceiverClientManager(conf));
++ scmClient = new ContainerOperationClient(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testNodeCanBeDecommMaintAndRecommissioned()
+ throws IOException {
+ NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager();
+
+ List<DatanodeDetails> dns = nm.getAllNodes();
+ scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // Ensure one node is decommissioning
+ List<DatanodeDetails> decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(1, decomNodes.size());
+
+ scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // Ensure zero nodes are now decommissioning
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+
+ scmClient.startMaintenanceNodes(Arrays.asList(
+ getDNHostAndPort(dns.get(0))), 10);
+
+ // None are decommissioning
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+
+ // One is in Maintenance
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(1, decomNodes.size());
+
+ scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0))));
+
+ // None are in maintenance
+ decomNodes = nm.getNodes(
+ HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE,
+ HddsProtos.NodeState.HEALTHY);
+ assertEquals(0, decomNodes.size());
+ }
+
+ private String getDNHostAndPort(DatanodeDetails dn) {
+ return dn.getHostName()+":"+dn.getPorts().get(0).getValue();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org