You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/05/11 23:29:59 UTC
[ozone] 01/02: Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade-master-merge2
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 253cc5540f65e15f674f0f83e9c1bc395005cda6
Merge: cdb1471 48b84c5
Author: Ethan Rose <er...@apache.org>
AuthorDate: Mon May 3 18:20:08 2021 -0400
Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-master-merge2
* upstream/master: (56 commits)
HDDS-2212. Genconf tool should generate config files for secure clust… (#1788)
HDDS-5166. Remove duplicate assignment of OZONE_OPTS for freon and sh (#2195)
Revert "HDDS-5144. Create github check to alert when dependency tree is changed (#2177)"
HDDS-4983. Display key offset for each block in command key info (#2051)
HDDS-5144. Create github check to alert when dependency tree is changed (#2177)
HDDS-4585. Support bucket acl operation in S3g (#1701)
HDDS-5153. Decommissioning a dead node should complete immediately (#2190)
HDDS-5147. Intermittent test failure in TestContainerDeletionChoosingPolicy#testRandomChoosingPolicy (#2188)
HDDS-5152. Fix Suggested leader in Client. (#2189)
HDDS-5148. Bump ratis version to 2.1.0-ff8aa66-SNAPSHOT (#2184)
HDDS-4515. Datanodes should be able to persist and load CRL (#2181)
HDDS-5060. [SCM HA Security] Make InterSCM grpc channel secure. (#2187)
HDDS-5051. Ensure failover to suggested leader if any for NotLeaderException. (#2141)
HDDS-5127. Fix getServiceList when SCM HA is enabled (#2173)
HDDS-4889. Add simple CI check for docs (#2156)
HDDS-5131. Use timeout in github actions (#2176)
HDDS-5103. Fix Install Snapshot Mechanism in SCMStateMachine. (#2155)
HDDS-5124. Use OzoneConsts.OZONE_TIME_ZONE instead of "GMT" (#2166)
HDDS-5047. Refactor Pipeline to use ReplicationConfig instead of factor/type (#2096)
HDDS-5083. Bump version of common-compress (#2139)
...
Conflicts:
hadoop-hdds/common/pom.xml
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
.github/workflows/cancel-ci.yaml | 41 ++
.github/workflows/post-commit.yml | 181 ++---
.gitignore | 2 +
HISTORY.md | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 +
.../hadoop/hdds/scm/storage/BlockInputStream.java | 7 +-
.../storage/DummyBlockInputStreamWithRetry.java | 7 +-
hadoop-hdds/common/pom.xml | 12 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 104 ++-
.../hdds/ratis/ServerNotLeaderException.java | 113 +++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 -
.../hadoop/hdds/scm/ha/NonRetriableException.java | 23 +-
.../scm/ha/RetriableWithNoFailoverException.java | 25 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 129 +++-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 5 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 67 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 13 +-
.../hadoop/hdds/security/x509/crl}/CRLCodec.java | 2 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 25 +-
.../hdds/security/x509/crl}/CRLInfoCodec.java | 3 +-
.../hdds/security/x509/crl/package-info.java | 23 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 0
.../apache/hadoop/hdds/utils/db/package-info.java | 23 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 +
.../org/apache/hadoop/ozone/common/Storage.java | 12 +
.../ozone/container/common/helpers/BlockData.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 154 +++--
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 22 +-
.../hdds/ratis/TestServerNotLeaderException.java | 82 +++
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 12 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 12 +-
.../hdds/datanode/metadata/CRLDBDefinition.java | 99 +++
.../hdds/datanode/metadata/DatanodeCRLStore.java | 84 +++
.../datanode/metadata/DatanodeCRLStoreImpl.java | 127 ++++
.../hdds/datanode/metadata/package-info.java | 23 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 23 +-
.../common/statemachine/DatanodeStateMachine.java | 2 +-
.../common/states/datanode/InitDatanodeState.java | 4 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 58 +-
.../container/keyvalue/helpers/BlockUtils.java | 27 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 16 +-
.../container/metadata/AbstractDatanodeStore.java | 26 +-
.../metadata/TestDatanodeCRLStoreImpl.java | 127 ++++
.../hdds/datanode/metadata/package-info.java | 27 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 6 +-
.../container/common/helpers/TestBlockData.java | 2 +-
.../common/impl/TestContainerDataYaml.java | 13 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 31 +-
.../container/keyvalue/TestKeyValueContainer.java | 43 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 18 +-
.../container/ozoneimpl/TestContainerReader.java | 1 +
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 13 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 162 +++++
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds}/ExitManager.java | 2 +-
.../java/org/apache/hadoop/hdds}/NodeDetails.java | 2 +-
.../java/org/apache/hadoop/hdds/package-info.java | 17 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 15 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 37 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 11 +
.../scm/protocol/ScmBlockLocationProtocol.java | 25 +-
...lockLocationProtocolClientSideTranslatorPB.java | 53 +-
...inerLocationProtocolClientSideTranslatorPB.java | 1 +
.../SCMBlockLocationFailoverProxyProvider.java | 59 +-
.../hadoop/hdds/scm/proxy/SCMClientConfig.java | 27 +-
.../SCMContainerLocationFailoverProxyProvider.java | 69 +-
.../SCMSecurityProtocolFailoverProxyProvider.java | 79 ++-
.../CRLInfo.java => certificate/CertInfo.java} | 97 +--
.../certificate/authority/CertificateServer.java | 15 +
.../certificate/authority/CertificateStore.java | 28 +
.../certificate/authority/DefaultApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 28 +-
.../certificate/authority/DefaultCRLApprover.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 17 +
.../client/DefaultCertificateClient.java | 38 +-
.../security/x509/certificate/package-info.java | 18 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 25 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 98 ++-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 4 +-
.../x509/certificate/authority/MockCAStore.java | 18 +
.../certificate/authority/TestDefaultCAServer.java | 10 +
.../x509/certificate/utils/TestCRLCodec.java | 1 +
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
hadoop-hdds/interface-client/pom.xml | 4 +-
.../interface-client/src/main/proto/hdds.proto | 9 +
hadoop-hdds/interface-server/pom.xml | 4 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 29 +
hadoop-hdds/pom.xml | 10 +-
hadoop-hdds/server-scm/pom.xml | 8 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 45 +-
.../hdds/scm/container/ContainerManagerImpl.java | 18 +-
.../hdds/scm/container/ContainerManagerV2.java | 9 +-
.../hdds/scm/container/ContainerStateManager.java | 31 +-
.../hdds/scm/container/SCMContainerManager.java | 6 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 39 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 28 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 23 +
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 +
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 19 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 6 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 48 +-
.../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 2 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 36 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 29 +-
.../{CRLInfoCodec.java => CertInfoCodec.java} | 22 +-
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 19 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 14 +-
.../hdds/scm/node/NodeDecommissionManager.java | 57 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 62 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 41 +-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 38 +-
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 24 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 39 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 74 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 18 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 29 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 12 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 34 +-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 35 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 163 +----
.../hdds/scm/pipeline/RatisPipelineProvider.java | 54 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 14 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 74 +-
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 4 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 27 +-
.../hadoop/hdds/scm/pipeline/StateManager.java | 42 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 83 ++-
...lockLocationProtocolServerSideTranslatorPB.java | 20 +-
...inerLocationProtocolServerSideTranslatorPB.java | 9 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 20 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 15 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 106 ++-
.../hdds/scm/server/SCMClientProtocolServer.java | 12 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 13 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 19 +-
.../hdds/scm/server/StorageContainerManager.java | 117 ++--
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 31 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 21 +-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 21 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 95 +--
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 12 +-
.../container/TestCloseContainerEventHandler.java | 10 +-
.../scm/container/TestContainerManagerImpl.java | 18 +-
.../scm/container/TestContainerStateManager.java | 15 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 67 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 28 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 53 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 21 +-
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 83 +--
.../scm/pipeline/TestPipelinePlacementPolicy.java | 10 +-
.../scm/pipeline/TestPipelineStateManager.java | 101 +--
.../scm/pipeline/TestRatisPipelineProvider.java | 100 +--
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 71 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 +
.../scm/pipeline/TestSimplePipelineProvider.java | 28 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 26 +-
.../TestOneReplicaPipelineSafeModeRule.java | 13 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 18 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 36 +-
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 6 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 3 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 4 +-
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 15 +-
.../hadoop/ozone/client/OzoneKeyLocation.java | 15 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 57 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 76 ++-
.../hadoop/ozone/client/TestHddsClientUtils.java | 150 ++--
hadoop-ozone/common/pom.xml | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 59 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 +
.../ozone/security/acl/IAccessAuthorizer.java | 1 -
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 7 +-
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 16 +-
hadoop-ozone/dev-support/checks/_lib.sh | 142 ++++
hadoop-ozone/dev-support/checks/acceptance.sh | 5 +
hadoop-ozone/dev-support/checks/bats.sh | 5 +
.../dev-support/checks/{bats.sh => docs.sh} | 21 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 12 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 16 +
.../dist/dev-support/bin/dist-layout-stitching | 3 +
hadoop-ozone/dist/pom.xml | 4 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +
.../src/main/compose/compatibility/docker-config | 1 +
.../dist/src/main/compose/ozone-csi/docker-config | 1 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozoneblockade/docker-config | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 1 +
.../src/main/compose/ozonescripts/docker-config | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 26 +
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 1 -
.../src/main/compose/ozonesecure/docker-config | 1 +
.../dist/src/main/compose/ozonesecure/test.sh | 51 +-
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 1 -
.../compose/upgrade/compose/non-ha/docker-config | 1 +
.../dist/src/main/compose/xcompat/docker-config | 1 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 15 +-
.../src/main/license/bin/licenses/LICENSE-CDDL.txt | 133 ++++
...-jakarta.validation-jakarta.validation-api.txt} | 241 -------
.../LICENSE-jakarta.ws.rs-jakarta.ws.rs-api.md | 637 +++++++++++++++++
.../LICENSE-javax.ws.rs-javax.ws.rs-api.txt | 759 ---------------------
.../NOTICE-jakarta.ws.rs-jakarta.ws.rs-api.md | 61 ++
.../src/main/smoketest/admincli/pipeline.robot | 7 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 2 -
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
.../network-tests/src/test/compose/docker-config | 1 +
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 12 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 1 -
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 11 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../metrics/TestSCMContainerManagerMetrics.java | 9 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 25 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 9 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 7 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 10 +-
.../TestRatisPipelineCreateAndDestroy.java | 30 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 16 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 15 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 15 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 1 -
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 8 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 6 +-
.../hadoop/ozone/TestStorageContainerManager.java | 6 +
.../ozone/client/CertificateClientTestImpl.java | 13 +
.../TestContainerStateMachineFailureOnRead.java | 7 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 5 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 25 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 120 +++-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 5 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 13 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 1 -
.../ozone/om/TestOzoneManagerRestInterface.java | 3 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 13 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 5 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 147 ++++
.../TestSCMContainerPlacementPolicyMetrics.java | 7 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 87 +--
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 9 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 7 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 11 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
hadoop-ozone/interface-storage/pom.xml | 4 +-
hadoop-ozone/ozone-manager/pom.xml | 4 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 17 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 81 ++-
.../hadoop/ozone/om/OzoneManagerStarter.java | 4 -
.../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 2 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 12 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 2 +
.../ozone/om/request/TestOMRequestUtils.java | 54 +-
.../om/request/key/TestOMKeyCommitRequest.java | 78 ++-
.../ozone/om/request/key/TestOMKeyRequest.java | 6 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 4 +-
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 4 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 6 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 2 -
.../hadoop/ozone/recon/api/NodeEndpoint.java | 6 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 11 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 9 +-
.../recon/scm/ReconPipelineReportHandler.java | 2 +-
.../scm/ReconStorageContainerManagerFacade.java | 15 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 9 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 12 +-
hadoop-ozone/s3gateway/pom.xml | 20 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 283 +++++++-
.../s3/endpoint/ListMultipartUploadsResult.java | 53 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 +-
.../endpoint/PutBucketAclRequestUnmarshaller.java | 85 +++
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 342 ++++++++++
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 235 +++++++
.../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 72 ++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +
.../apache/hadoop/ozone/s3/util/RFC1123Util.java | 3 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 23 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 24 +
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 266 ++++++++
.../{TestBucketGet.java => TestBucketList.java} | 76 +--
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 6 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 56 +-
.../src/test/resources/groupAccessControlList.xml | 57 ++
.../src/test/resources/userAccessControlList.xml | 42 ++
hadoop-ozone/tools/pom.xml | 12 +-
.../StorageContainerManagersCommandHandler.java | 2 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 5 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 3 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 5 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 6 +-
.../GenerateOzoneRequiredConfigurations.java | 44 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 92 +--
.../ozone/genesis/BenchMarkOzoneManager.java | 5 +-
.../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 7 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 5 +-
.../TestGenerateOzoneRequiredConfigurations.java | 52 +-
pom.xml | 42 +-
338 files changed, 7520 insertions(+), 3682 deletions(-)
diff --cc hadoop-hdds/common/pom.xml
index 4d6a0a2,cebee64..6b36972
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@@ -207,18 -201,10 +206,17 @@@ https://maven.apache.org/xsd/maven-4.0.
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.2</version>
+ <scope>test</scope>
+ <!-- Needed for mocking RaftServerImpl -->
+ </dependency>
+ <dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
- <scope>compile</scope>
+ <scope>test</scope>
</dependency>
-
</dependencies>
<build>
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 997b65d,460803e..cf44a92
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@@ -440,7 -444,10 +444,11 @@@ public final class OzoneConfigKeys
public static final boolean OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT =
false;
+ public static final int OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT = -1;
+ public static final String OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY =
+ "ozone.client.key.provider.cache.expiry";
+ public static final long OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
+ TimeUnit.DAYS.toMillis(10); // 10 days
/**
* There is no need to instantiate this class.
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index dd58cb8,efaeb23..5a9e71a
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -435,6 -434,11 +439,14 @@@ public final class OzoneConsts
public static final String SCM_SUB_CA_PREFIX = "scm-sub@";
public static final String SCM_ROOT_CA_PREFIX = "scm@";
+ // Layout Version written into Meta Table ONLY during finalization.
+ public static final String LAYOUT_VERSION_KEY = "#LAYOUTVERSION";
++
+ // Kerberos constants
+ public static final String KERBEROS_CONFIG_VALUE = "kerberos";
+ public static final String HTTP_AUTH_TYPE_SUFFIX = "http.auth.type";
+ public static final String OZONE_SECURITY_ENABLED_SECURE = "true";
+ public static final String OZONE_HTTP_SECURITY_ENABLED_SECURE = "true";
+ public static final String OZONE_HTTP_FILTER_INITIALIZERS_SECURE =
+ "org.apache.hadoop.security.AuthenticationFilterInitializer";
}
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index c9a407e,baa5889..d45ddcc
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@@ -70,9 -71,7 +72,8 @@@ import com.google.common.base.Precondit
import com.sun.jmx.mbeanserver.Introspector;
import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate;
import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString;
- import static org.apache.hadoop.hdds.utils.HAUtils.checkSecurityAndSCMHAEnabled;
import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
+import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
import static org.apache.hadoop.util.ExitUtil.terminate;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import org.slf4j.Logger;
@@@ -246,12 -245,10 +247,16 @@@ public class HddsDatanodeService extend
}
LOG.info("Hdds Datanode login successful.");
}
+
+ DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf,
+ datanodeDetails.getUuidString());
+ if (layoutStorage.getState() != INITIALIZED) {
+ layoutStorage.initialize();
+ }
+
+ // initialize datanode CRL store
+ dnCRLStore = new DatanodeCRLStoreImpl(conf);
+
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
initializeCertificateClient(conf);
}
diff --cc hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 7d746ad,e7e4502..14a2665
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@@ -336,5 -323,13 +336,14 @@@ message UpgradeFinalizationStatus
message CRLInfoProto {
required string x509CRL = 1;
required uint64 creationTimestamp = 2;
+ required int64 crlSequenceID = 3;
+ }
+
+ /**
+ * Information for X509 Certificate.
+ */
+ message CertInfoProto {
+ required string x509Certificate = 1;
+ required uint64 timestamp = 2;
}
+
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
index 05a130e,b7d8864..a0f7e0f
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
@@@ -152,17 -151,9 +156,17 @@@ public class PipelineManagerV2Impl impl
throw new IOException("Pipeline creation is not allowed as safe mode " +
"prechecks have not yet passed");
}
+
+ if (freezePipelineCreation.get()) {
+ String message = "Cannot create new pipelines while pipeline creation " +
+ "is frozen.";
+ LOG.info(message);
+ throw new IOException(message);
+ }
+
lock.lock();
try {
- Pipeline pipeline = pipelineFactory.create(type, factor);
+ Pipeline pipeline = pipelineFactory.create(replicationConfig);
stateManager.addPipeline(pipeline.getProtobufMessage(
ClientVersions.CURRENT_VERSION));
recordMetricsForPipeline(pipeline);
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index cdc6e48,c96ce2d..ddf0d50
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@@ -275,15 -270,9 +277,15 @@@ public class SCMPipelineManager impleme
throw new IOException("Pipeline creation is not allowed as safe mode " +
"prechecks have not yet passed");
}
+ if (freezePipelineCreation.get()) {
+ LOG.debug("Pipeline creation is frozen while an upgrade is in " +
+ "progress");
+ throw new IOException("Pipeline creation is frozen while an upgrade " +
+ "is in progress");
+ }
lock.writeLock().lock();
try {
- Pipeline pipeline = pipelineFactory.create(type, factor);
+ Pipeline pipeline = pipelineFactory.create(replicationConfig);
if (pipelineStore != null) {
pipelineStore.put(pipeline.getId(), pipeline);
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java
index 55f7f3e,4459902..6b8f192
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java
@@@ -28,8 -27,8 +28,10 @@@ import java.io.IOException
import java.util.Properties;
import java.util.UUID;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
+ import static org.apache.hadoop.ozone.OzoneConsts.SCM_HA;
+ import static org.apache.hadoop.ozone.OzoneConsts.STORAGE_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.PRIMARY_SCM_NODE_ID;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CERT_SERIAL_ID;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 03272a5,453d421..143077a
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -93,20 -99,12 +100,14 @@@ import org.apache.hadoop.hdds.scm.pipel
import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
+import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizer;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
- import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer;
- import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile;
- import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
- import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
- import org.apache.hadoop.hdds.utils.HAUtils;
- import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.IOUtils;
diff --cc hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
index 744e458,e55477d..a6d4482
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
@@@ -26,13 -24,13 +26,14 @@@ OZONE-SITE.XML_ozone.scm.pipeline.owner
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.client.address=scm
+
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_ozone.recon.address=recon:9891
+
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
+ OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 9270e1c,2da2211..dbcc173
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@@ -241,12 -240,12 +243,12 @@@ public class MiniOzoneClusterImpl imple
*/
@Override
public void waitForPipelineTobeReady(HddsProtos.ReplicationFactor factor,
- int timeoutInMs) throws
- TimeoutException, InterruptedException {
+ int timeoutInMs) throws
+ TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> {
int openPipelineCount = scm.getPipelineManager().
- getPipelines(HddsProtos.ReplicationType.RATIS,
- factor, Pipeline.PipelineState.OPEN).size();
+ getPipelines(new RatisReplicationConfig(factor),
+ Pipeline.PipelineState.OPEN).size();
return openPipelineCount >= 1;
}, 1000, timeoutInMs);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
index 676598d,3f9d2de..950d3c5
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
@@@ -27,10 -27,6 +27,9 @@@ import org.apache.hadoop.hdds.conf.Ozon
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+
- import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
+import static org.apache.hadoop.ozone.om.OmGenericConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
/**
* OMStorage is responsible for management of the StorageDirectories used by
@@@ -47,19 -43,9 +46,11 @@@ public class OMStorage extends Storage
* @throws IOException if any directories are inaccessible.
*/
public OMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR);
+ super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR,
+ getInitLayoutVersion(conf, OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION,
+ OMLayoutVersionManager::maxLayoutVersion));
}
- public void setScmId(String scmId) throws IOException {
- if (getState() == StorageState.INITIALIZED) {
- throw new IOException("OM is already initialized.");
- } else {
- getStorageInfo().setProperty(SCM_ID, scmId);
- }
- }
-
public void setOmCertSerialId(String certSerialId) throws IOException {
getStorageInfo().setProperty(OM_CERT_SERIAL_ID, certSerialId);
}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 9fd34fe,6c18e0e..f9ee84f
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -161,9 -159,7 +162,9 @@@ import org.apache.hadoop.ozone.security
import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import org.apache.hadoop.ozone.security.acl.RequestContext;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
+import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
- import org.apache.hadoop.ozone.util.ExitManager;
+ import org.apache.hadoop.hdds.ExitManager;
import org.apache.hadoop.ozone.util.OzoneVersionInfo;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
index c25bbf1,99dc3e6..d6a9550
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
@@@ -155,17 -130,8 +152,16 @@@ public class OzoneManagerStarter extend
@Override
public boolean init(OzoneConfiguration conf) throws IOException,
AuthenticationException {
- checkSecurityAndSCMHAEnabled(conf);
return OzoneManager.omInit(conf);
}
- }
+ @Override
+ public void startAndCancelPrepare(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
+ OzoneManager om = OzoneManager.createOm(conf);
+ om.getPrepareState().cancelPrepare();
+ om.start();
+ om.join();
+ }
+ }
}
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index 2720a30,2272448..b82364f
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@@ -57,7 -56,7 +56,8 @@@ import org.apache.hadoop.hdds.scm.safem
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+ import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.io.IOUtils;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org