You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by li...@apache.org on 2020/07/15 11:54:01 UTC
[hadoop-ozone] branch HDDS-2823 updated (8a8c9eb -> 40127b3)
This is an automated email from the ASF dual-hosted git repository.
licheng pushed a change to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 8a8c9eb HDDS-3191: switch from SCMPipelineManager to PipelineManagerV2Impl (#1151)
add 04f8e6a HDDS-3884. Fix typo in pom.xml. (#1138)
add 048ccfa HDDS-3151. Disable flaky test: TestCloseContainerHandlingByClient#testMultiBlockWrites3
add 1d7fd3a HDDS-3887. Remove redundant code for HealthyPipelineSafeModeRule (#1140)
add 4daef59 HDDS-3888. OneReplicaPipelineSafeModeRule Unnecessary logic (#1141)
add 95073f4 HDDS-3865. Export the SCM client IPC port in docker-compose (#1124)
add bf23dcb HDDS-3868. Implement getTrashRoot and getTrashRoots in o3fs (#1134)
add 3479e67 HDDS-3699. Change write chunk failure logging level to ERROR in BlockOutputStream. (#1006)
add 574763b HDDS-3770. Improve getPipelines performance (#1066)
add f734182 HDDS-3782. Remove podAntiAffinity from datanode-statefulset (#1057)
add dd46a55 HDDS-3161. Block illegal characters when creating keys. (#812)
add 4e6d6c6 HDDS-3876. Display summary of failures as a separate job step (#1131)
add 3c1d8b6 HDDS-3246. Include OM hostname info in getserviceroles subcommand of OM CLI (#706)
add 2fb4a40 HDDS-2978. Intermittent failure in TestResourceLimitCache (#1146)
add 1d13b4f HDDS-3803. [OFS] Add User Guide (#1144)
add 8ff5c8e HDDS-3831. Enhance Recon ContainerEndPoint to report on different unhealty container states (#1148)
add 9e0f5ba HDDS-3440. Enable TestFreonWithPipelineDestroy test cases (#1153)
add 0300feb HDDS-2413. Set configuration variables from annotated java objects (#1106)
add c890901 HDDS-3908. Duplicate dot in Prometheus endpoint config name (#1155)
add b1ab8bc HDDS-3632. starter scripts can't manage Ozone and HDFS datandodes on the same machine (#1115)
add bfb9a15 HDDS-3875. Package classpath files to the jar files instead of uploading them as artifacts (#1133)
add f6e3ff2 HDDS-3421. Enabling TestStorageContainerManager. (#1127)
add 3aa2774 HDDS-3891. Add the usage of ofs in doc. (#1143)
add b4798cd HDDS-3737. Avoid serialization between UUID and String (#1053)
add 86bd3b3 HDDS-3862. Prepare checks for running some tests multiple times (#1128)
add 6705761 HDDS-3917. Add recon to no_proxy of docker-config for acceptance test (#1161)
add 2c4e656 Revert "HDDS-3421. Enabling TestStorageContainerManager. (#1127)"
add 6eebf2f HDDS-3900: Update default value of 'ozone.om.ratis.segment.size' and 'preallocated.size' to improve OM write perf (#1160)
add fb7f671 HDDS-3863. Provide script to re-generate Kubernetes resources files (#1123)
add a84d5bc HDDS-3604. Use Ozone version of Hadoop Security/Token classes (#933)
add 9aed77c HDDS-3928. Fix endpoint display in S3 Gateway webpage (#1167)
add a4ef847 HDDS-3929. Prettify OMDeleteRequest error log. (#1168)
add 59a3c2e HDDS-3913. Recon build should ignore proxies (#1159)
add 267fd9e HDDS-3910. JooqCodeGenerator interrupted but still alive (#1170)
add 0176bb1 HDDS-3931. Maven warning due to deprecated expression pom.artifactId (#1172)
add 8704365 HDDS-3705. [OFS] Implement getTrashRoots for trash cleanup (#1089)
add 471038f HDDS-3909. Use ContainerScrubberConfiguration in ChunkManagerFactory (#1158)
add 2e9feff HDDS-3932. Hide jOOQ logo message from the log output on compile (#1171)
add db7e236 HDDS-3939. Update proto.lock files (#1178)
add 0408dd6 HDDS-3935. Change latest snapshot log to debug (#1176)
add 25d3e52 HDDS-3940. Update angular to v1.7.9 (#1180)
add f7a0d07 HDDS-3879. Introduce SCM and OM layoutVersion zero to the VERSION file (#1156)
add d85c7e3 HDDS-3747. Redundancy if condition code in ListPipelinesSubcommand (#1051)
add 55a7bc8 HDDS-3944. OM StateMachine unpause fails with NPE (#1183)
add d6693a9 HDDS-3918. ConcurrentModificationException in ContainerReportHandler.… (#1174)
add 2af6198 HDDS-3914. Remove LevelDB configuration option for DN Metastore (#1166)
add 5977168 HDDS-3921. IllegalArgumentException triggered in SCMContainerPlacemen… (#1162)
add 7054b4b HDDS-3937. Update jquery to v3.5.1 (#1177)
add c9f6365 HDDS-3941. Enable core dump when crash in C++ (#1186)
add 9106f14 HDDS-3920. Too many redudant replications due to fail to get node's a… (#1163)
add 92926ae HDDS-3951. Rename the num.write.chunk.thread key. (#1187)
add 90e8211 HDDS-3685. Remove replay logic from actual request logic. (#1082)
add 537172e HDDS-3861. Fix handlePipelineFailure throw exception if role is follower (#1122)
add c6b45ba HDDS-3789. Fix TestOzoneRpcClientAbstract#testDeletedKeyForGDPR. (#1198)
add 8bb452f HDDS-3062. Fix TestOzoneRpcClientAbstract.testListVolume. (#1194)
add 415d4aa HDDS-3062. Fix TestOzoneRpcClientAbstract.testListVolume - addendum for checkstyle
add 6d979e4 HDDS-3957. Fix mixed use of Longs.toByteArray and Ints.fromByteArray (#1199)
add 34404f3 HDDS-3930. Fix OMKeyDeletesRequest. (#1195)
add 4d964f3 HDDS-3798. Display version and setupTime of DN in recon web (#1136)
add 8573e5b HDDS-3060. /retest github comment does not work (#1137)
add 0a1cce5 HDDS-3925. SCM Pipeline DB should directly use UUID bytes for key rather than rely on proto serialization for key. (#1197)
add cbd75cb HDDS-3885. Create Datanode home page (#1139)
add 1068601 HDDS-2768. security/SecuringS3.md (#1188)
add 9260484 HDDS-2769. security/SecurityWithRanger.md (#1189)
new 40127b3 Merge branch 'master' into HDDS-2823
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/comment-commands/retest.sh | 66 +++-
.github/workflows/post-commit.yml | 18 ++
.gitignore | 2 +
LICENSE.txt | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 20 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +-
hadoop-hdds/common/src/main/conf/hadoop-env.sh | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 -
.../hadoop/hdds/conf/HddsPrometheusConfig.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 21 +-
.../java/org/apache/hadoop/hdds/fs/DUFactory.java | 19 +-
.../hdds/fs/DedicatedDiskSpaceUsageFactory.java | 3 +-
.../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 3 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 123 +++++++-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 3 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../scm/container/common/helpers/ExcludeList.java | 37 +--
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 28 +-
.../hadoop/hdds/scm/pipeline/PipelineID.java | 23 +-
.../utils/LegacyHadoopConfigurationSource.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 25 +-
.../org/apache/hadoop/ozone/common/Storage.java | 7 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 24 +-
.../common/src/main/resources/ozone-default.xml | 50 +--
.../hadoop/hdds/conf/SimpleConfiguration.java | 2 +-
.../hdds/conf/SimpleConfigurationParent.java | 6 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 85 ++++-
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 14 +-
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 14 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 80 ++++-
.../hadoop/hdds/conf/ConfigurationSource.java | 7 +-
.../hadoop/hdds/conf/ConfigurationTarget.java | 54 ++++
.../hdds/conf/MutableConfigurationSource.java} | 11 +-
hadoop-hdds/container-service/pom.xml | 31 ++
.../java/org/apache/hadoop/ozone/DNMXBean.java | 8 +-
...sDatanodeStopService.java => DNMXBeanImpl.java} | 15 +-
.../hadoop/ozone/HddsDatanodeHttpServer.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 38 ++-
.../container/common/helpers/DatanodeIdYaml.java | 3 +-
.../RandomContainerDeletionChoosingPolicy.java | 7 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 3 +-
.../transport/server/ratis/XceiverServerRatis.java | 14 +-
.../container/keyvalue/KeyValueContainer.java | 5 -
.../container/keyvalue/KeyValueContainerCheck.java | 8 +-
.../container/keyvalue/KeyValueContainerData.java | 17 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 3 +-
.../keyvalue/impl/ChunkManagerFactory.java | 11 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 4 +
.../main/resources/webapps/hddsDatanode/.gitkeep | 17 -
.../webapps/hddsDatanode/dn-overview.html | 8 +-
.../src/main/resources/webapps/hddsDatanode/dn.js} | 21 +-
.../resources/webapps/hddsDatanode}/index.html | 14 +-
.../main/resources/webapps/hddsDatanode}/main.html | 4 +-
.../ozone/container/common/ContainerTestUtils.java | 3 +-
.../container/common/TestBlockDeletingService.java | 10 +-
.../container/common/TestDatanodeStateMachine.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../common/report/TestReportPublisher.java | 3 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 16 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 2 +-
.../ozoneimpl/TestContainerScrubberMetrics.java | 5 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +-
hadoop-hdds/docs/content/design/ofs.md | 143 ++++++++-
hadoop-hdds/docs/content/interface/OzoneFS.md | 58 +++-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 52 ++-
...ecuityWithRanger.md => SecuityWithRanger.zh.md} | 18 +-
hadoop-hdds/docs/content/security/SecuringS3.md | 6 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 50 +++
hadoop-hdds/docs/pom.xml | 2 +-
.../themes/ozonedoc/layouts/partials/footer.html | 2 +-
.../themes/ozonedoc/static/js/jquery-3.4.1.min.js | 2 -
.../themes/ozonedoc/static/js/jquery-3.5.1.min.js | 2 +
.../hdds/conf/DatanodeRatisServerConfig.java | 36 ++-
.../hadoop/hdds/server/http/BaseHttpServer.java | 10 +-
.../apache/hadoop/hdds/server/http/HttpConfig.java | 6 +-
.../hadoop/hdds/server/http/HttpServer2.java | 7 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 1 +
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 1 +
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 25 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 16 +
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 2 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 8 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 5 +
.../resources/webapps/static/angular-1.6.4.min.js | 332 -------------------
.../resources/webapps/static/angular-1.7.9.min.js | 350 +++++++++++++++++++++
.../webapps/static/angular-route-1.6.4.min.js | 17 -
.../webapps/static/angular-route-1.7.9.min.js | 17 +
.../resources/webapps/static/jquery-3.4.1.min.js | 2 -
.../resources/webapps/static/jquery-3.5.1.min.js | 2 +
.../hadoop/hdds/utils/TestMetadataStore.java | 48 +--
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 3 -
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 224 +++++++++++++
hadoop-hdds/interface-client/pom.xml | 2 +-
.../interface-client/src/main/proto/hdds.proto | 19 +-
.../interface-client/src/main/proto/proto.lock | 42 ++-
hadoop-hdds/pom.xml | 32 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 3 +-
.../hdds/scm/container/ContainerReportHandler.java | 9 +-
.../IncrementalContainerReportHandler.java | 13 +-
.../hdds/scm/container/ReplicationManager.java | 27 +-
.../hdds/scm/container/SCMContainerManager.java | 7 +-
.../hadoop/hdds/scm/metadata/PipelineIDCodec.java | 38 ++-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 69 ++--
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 49 ++-
.../hdds/scm/pipeline/SCMPipelineManager.java | 54 +++-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 5 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 5 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +-
.../server/StorageContainerManagerHttpServer.java | 6 +-
.../src/main/resources/webapps/scm/index.html | 6 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../TestIncrementalContainerReportHandler.java | 36 ++-
.../hdds/scm/container/TestReplicationManager.java | 4 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 144 +++++++++
.../hdds/scm/node/TestNodeReportHandler.java | 5 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 115 +++++++
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 7 +-
.../hdds/scm/cli/ContainerOperationClient.java | 4 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 21 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 13 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 3 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 32 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 22 ++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 20 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 9 +-
.../hadoop/ozone/om/exceptions/OMException.java | 2 +-
.../ozone/om/exceptions/OMReplayException.java | 50 ---
.../ozone/om/ha/OMFailoverProxyProvider.java | 2 +-
.../hadoop/ozone/om/helpers/OmDeleteKeys.java} | 35 ++-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 5 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 30 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 2 +-
.../hadoop/ozone/protocolPB/OzonePBHelper.java | 60 +++-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 12 +-
.../dev-support/checks/_mvn_unit_report.sh | 14 +-
hadoop-ozone/dev-support/checks/integration.sh | 25 +-
.../checks/{integration.sh => junit.sh} | 49 ++-
hadoop-ozone/dev-support/checks/unit.sh | 24 +-
hadoop-ozone/dist/pom.xml | 132 +-------
.../src/main/compose/ozone-csi/docker-compose.yaml | 1 +
.../dist/src/main/compose/ozone-csi/docker-config | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop31/docker-config | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop32/docker-config | 2 +-
.../compose/ozone-om-ha-s3/docker-compose.yaml | 1 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 1 +
.../src/main/compose/ozone-topology/docker-config | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 1 +
.../dist/src/main/compose/ozone/docker-config | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 1 +
.../src/main/compose/ozoneblockade/docker-config | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 2 +-
.../main/compose/ozonescripts/docker-compose.yaml | 1 +
.../src/main/compose/ozonescripts/docker-config | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 2 +-
.../compose/ozonesecure-om-ha/docker-compose.yaml | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 1 +
.../config.yaml => definitions/onenode.yaml} | 15 +-
.../main/k8s/examples/getting-started/Flekszible | 1 +
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../startOM.sh => k8s/examples/regenerate-all.sh} | 21 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 6 +-
.../dist/src/shell/hdds/hadoop-functions.sh | 20 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 12 +-
.../src/test/compose/docker-compose.yaml | 1 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 72 ++++-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 158 +++++++++-
.../hadoop/fs/ozone/contract/OzoneContract.java | 15 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 15 +-
.../TestContainerStateManagerIntegration.java | 14 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 20 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 9 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 15 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 15 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 15 +-
.../rpc/TestCloseContainerHandlingByClient.java | 1 +
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 15 +-
.../rpc/TestContainerReplicationEndToEnd.java | 22 +-
.../TestContainerStateMachineFailureOnRead.java | 25 +-
.../rpc/TestContainerStateMachineFailures.java | 15 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 25 +-
.../client/rpc/TestFailureHandlingByClient.java | 15 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 15 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 15 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 5 -
.../client/rpc/TestValidateBCSIDOnRestart.java | 15 +-
.../ozone/client/rpc/TestWatchForCommit.java | 15 +-
.../hadoop/ozone/freon/TestDataValidate.java | 15 +-
.../freon/TestDataValidateWithDummyContainers.java | 4 -
.../ozone/freon/TestFreonWithDatanodeRestart.java | 15 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 32 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 15 +-
.../ozone/om/TestContainerReportWithKeys.java | 10 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 -
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 21 --
.../ozone/om/TestOzoneManagerHAWithData.java | 7 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 3 -
.../apache/hadoop/ozone/recon/TestReconTasks.java | 17 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../src/test/resources/ozone-site.xml | 2 +-
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../src/main/proto/OmClientProtocol.proto | 21 +-
.../interface-client/src/main/proto/Security.proto | 3 +-
.../interface-client/src/main/proto/proto.lock | 62 ++--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 4 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 24 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 8 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 36 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 89 +-----
.../om/request/bucket/OMBucketCreateRequest.java | 26 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 26 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 12 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 11 -
.../om/request/file/OMDirectoryCreateRequest.java | 47 +--
.../ozone/om/request/file/OMFileCreateRequest.java | 61 ++--
.../om/request/key/OMAllocateBlockRequest.java | 50 +--
.../ozone/om/request/key/OMKeyCommitRequest.java | 87 ++---
.../ozone/om/request/key/OMKeyCreateRequest.java | 60 ++--
.../ozone/om/request/key/OMKeyDeleteRequest.java | 36 +--
.../ozone/om/request/key/OMKeyPurgeRequest.java | 97 +-----
.../ozone/om/request/key/OMKeyRenameRequest.java | 130 +++-----
.../ozone/om/request/key/OMKeysDeleteRequest.java | 229 ++++++++------
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 24 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 6 -
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 6 -
.../om/request/key/acl/OMKeySetAclRequest.java | 6 -
.../request/key/acl/prefix/OMPrefixAclRequest.java | 30 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 6 -
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 6 -
.../key/acl/prefix/OMPrefixSetAclRequest.java | 6 -
.../S3InitiateMultipartUploadRequest.java | 5 -
.../multipart/S3MultipartUploadAbortRequest.java | 20 --
.../S3MultipartUploadCommitPartRequest.java | 34 +-
.../S3MultipartUploadCompleteRequest.java | 82 +----
.../security/OMCancelDelegationTokenRequest.java | 4 +-
.../security/OMGetDelegationTokenRequest.java | 6 +-
.../security/OMRenewDelegationTokenRequest.java | 5 +-
.../om/request/volume/OMVolumeCreateRequest.java | 23 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 -
.../om/request/volume/OMVolumeSetOwnerRequest.java | 57 +---
.../om/request/volume/OMVolumeSetQuotaRequest.java | 20 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 52 +--
.../request/volume/acl/OMVolumeAddAclRequest.java | 6 -
.../volume/acl/OMVolumeRemoveAclRequest.java | 6 -
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 -
.../hadoop/ozone/om/response/OMClientResponse.java | 2 +-
.../om/response/bucket/OMBucketCreateResponse.java | 2 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 2 +-
.../bucket/OMBucketSetPropertyResponse.java | 2 +-
.../response/bucket/acl/OMBucketAclResponse.java | 2 +-
.../response/file/OMDirectoryCreateResponse.java | 44 +--
.../om/response/file/OMFileCreateResponse.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 2 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 24 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 2 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 46 ++-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 11 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 59 +---
.../om/response/key/OMKeysDeleteResponse.java | 100 +++---
.../om/response/key/acl/OMKeyAclResponse.java | 2 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 2 +-
.../S3InitiateMultipartUploadResponse.java | 2 +-
.../multipart/S3MultipartUploadAbortResponse.java | 2 +-
.../S3MultipartUploadCommitPartResponse.java | 40 +--
.../S3MultipartUploadCompleteResponse.java | 48 +--
.../om/response/volume/OMVolumeAclOpResponse.java | 2 +-
.../om/response/volume/OMVolumeCreateResponse.java | 2 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 4 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 4 +-
.../main/resources/webapps/ozoneManager/index.html | 6 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 6 +-
.../request/bucket/TestOMBucketCreateRequest.java | 28 --
.../request/bucket/TestOMBucketDeleteRequest.java | 42 ---
.../bucket/TestOMBucketSetPropertyRequest.java | 29 --
.../om/request/file/TestOMFileCreateRequest.java | 29 --
.../ozone/om/request/key/TestOMKeyAclRequest.java | 9 +-
.../om/request/key/TestOMKeyCommitRequest.java | 79 -----
.../om/request/key/TestOMKeyCreateRequest.java | 42 ---
.../om/request/key/TestOMKeyDeleteRequest.java | 37 ---
.../key/TestOMKeyPurgeRequestAndResponse.java | 117 -------
.../om/request/key/TestOMKeyRenameRequest.java | 102 ------
.../om/request/key/TestOMKeysDeleteRequest.java | 155 +++++++++
.../om/request/key/TestOMPrefixAclRequest.java | 11 +-
.../security/TestOMGetDelegationTokenRequest.java | 3 +-
.../request/volume/TestOMVolumeCreateRequest.java | 26 --
.../request/volume/TestOMVolumeDeleteRequest.java | 38 ---
.../volume/TestOMVolumeSetOwnerRequest.java | 31 --
.../volume/TestOMVolumeSetQuotaRequest.java | 30 --
.../volume/acl/TestOMVolumeAddAclRequest.java | 33 --
.../volume/acl/TestOMVolumeRemoveAclRequest.java | 44 ---
.../volume/acl/TestOMVolumeSetAclRequest.java | 35 ---
.../file/TestOMDirectoryCreateResponse.java | 5 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 125 ++++++++
.../security/TestOMGetDelegationTokenResponse.java | 2 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 47 +++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 ++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 12 +
hadoop-ozone/pom.xml | 32 +-
.../ozone/recon/codegen/JooqCodeGenerator.java | 1 +
.../ozone/recon/codegen/ReconSqlDbConfig.java | 7 -
hadoop-ozone/recon/pom.xml | 2 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 -
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 124 +++++++-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 2 +
.../ozone/recon/api/types/DatanodeMetadata.java | 28 ++
...tadata.java => UnhealthyContainerMetadata.java} | 67 +++-
.../api/types/UnhealthyContainersResponse.java | 98 ++++++
...Report.java => UnhealthyContainersSummary.java} | 28 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 4 +-
.../recon/persistence/ContainerSchemaManager.java | 50 ++-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 4 +-
.../ReconIncrementalContainerReportHandler.java | 12 +-
.../recon/spi/ContainerDBServiceProvider.java | 8 -
.../spi/impl/ContainerDBServiceProviderImpl.java | 6 -
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 26 +-
.../webapps/recon/ozone-recon-web/api/db.json | 48 ++-
.../src/components/multiSelect/multiSelect.tsx | 5 +-
.../src/views/datanodes/datanodes.less | 14 +
.../src/views/datanodes/datanodes.tsx | 100 +++++-
.../ozone/recon/api/TestContainerEndpoint.java | 224 ++++++++++++-
.../hadoop/ozone/recon/api/TestEndpoints.java | 10 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
...TestReconIncrementalContainerReportHandler.java | 30 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 4 +-
.../s3gateway/src/main/resources/browser.html | 2 +-
.../src/main/resources/webapps/static/index.html | 2 +-
.../src/main/resources/webapps/static/s3g.js | 4 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 4 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 28 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 29 ++
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 6 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +-
pom.xml | 2 +-
363 files changed, 4976 insertions(+), 3876 deletions(-)
create mode 100644 hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationTarget.java
copy hadoop-hdds/config/src/{test/java/org/apache/hadoop/hdds/conf/package-info.java => main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java} (84%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java (84%)
copy hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/{HddsDatanodeStopService.java => DNMXBeanImpl.java} (72%)
delete mode 100644 hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep
copy hadoop-ozone/insight/dev-support/findbugsExcludeFile.xml => hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html (88%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java => container-service/src/main/resources/webapps/hddsDatanode/dn.js} (60%)
copy hadoop-hdds/{server-scm/src/main/resources/webapps/scm => container-service/src/main/resources/webapps/hddsDatanode}/index.html (88%)
copy hadoop-hdds/{server-scm/src/main/resources/webapps/scm => container-service/src/main/resources/webapps/hddsDatanode}/main.html (95%)
copy hadoop-hdds/docs/content/security/{SecuityWithRanger.md => SecuityWithRanger.zh.md} (54%)
create mode 100644 hadoop-hdds/docs/content/security/SecuringS3.zh.md
delete mode 100644 hadoop-hdds/docs/themes/ozonedoc/static/js/jquery-3.4.1.min.js
create mode 100644 hadoop-hdds/docs/themes/ozonedoc/static/js/jquery-3.5.1.min.js
delete mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/angular-1.6.4.min.js
create mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/angular-1.7.9.min.js
delete mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/angular-route-1.6.4.min.js
create mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/angular-route-1.7.9.min.js
delete mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/jquery-3.4.1.min.js
create mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/jquery-3.5.1.min.js
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIterator.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestPipelineIDCodec.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMReplayException.java
copy hadoop-ozone/{client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java => common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDeleteKeys.java} (61%)
copy hadoop-ozone/dev-support/checks/{integration.sh => junit.sh} (54%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/k8s/definitions/ozone/{transformations/config.yaml => definitions/onenode.yaml} (79%)
copy hadoop-ozone/dist/src/main/{compose/ozone-om-ha/startOM.sh => k8s/examples/regenerate-all.sh} (70%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{MissingContainerMetadata.java => UnhealthyContainerMetadata.java} (52%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UnhealthyContainersResponse.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{DatanodeStorageReport.java => UnhealthyContainersSummary.java} (64%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-2823
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
licheng pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 40127b3c2402a0cd279eded94764898a52a74c60
Merge: 8a8c9eb 9260484
Author: Li Cheng <ti...@tencent.com>
AuthorDate: Wed Jul 15 19:43:51 2020 +0800
Merge branch 'master' into HDDS-2823
.github/comment-commands/retest.sh | 66 +++-
.github/workflows/post-commit.yml | 18 ++
.gitignore | 2 +
LICENSE.txt | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 20 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +-
hadoop-hdds/common/src/main/conf/hadoop-env.sh | 3 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 -
.../hadoop/hdds/conf/HddsPrometheusConfig.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 21 +-
.../java/org/apache/hadoop/hdds/fs/DUFactory.java | 19 +-
.../hdds/fs/DedicatedDiskSpaceUsageFactory.java | 3 +-
.../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 3 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 123 +++++++-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 3 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../scm/container/common/helpers/ExcludeList.java | 37 +--
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 28 +-
.../hadoop/hdds/scm/pipeline/PipelineID.java | 23 +-
.../utils/LegacyHadoopConfigurationSource.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 25 +-
.../org/apache/hadoop/ozone/common/Storage.java | 7 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 24 +-
.../common/src/main/resources/ozone-default.xml | 50 +--
.../hadoop/hdds/conf/SimpleConfiguration.java | 2 +-
.../hdds/conf/SimpleConfigurationParent.java | 6 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 85 ++++-
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 14 +-
.../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 14 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 80 ++++-
.../hadoop/hdds/conf/ConfigurationSource.java | 7 +-
.../hadoop/hdds/conf/ConfigurationTarget.java | 54 ++++
.../hdds/conf/MutableConfigurationSource.java} | 12 +-
hadoop-hdds/container-service/pom.xml | 31 ++
.../org/apache/hadoop/ozone/DNMXBean.java} | 14 +-
.../org/apache/hadoop/ozone/DNMXBeanImpl.java} | 22 +-
.../hadoop/ozone/HddsDatanodeHttpServer.java | 7 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 38 ++-
.../container/common/helpers/DatanodeIdYaml.java | 3 +-
.../RandomContainerDeletionChoosingPolicy.java | 7 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 3 +-
.../transport/server/ratis/XceiverServerRatis.java | 14 +-
.../container/keyvalue/KeyValueContainer.java | 5 -
.../container/keyvalue/KeyValueContainerCheck.java | 8 +-
.../container/keyvalue/KeyValueContainerData.java | 17 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 3 +-
.../keyvalue/impl/ChunkManagerFactory.java | 11 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 4 +
.../webapps/hddsDatanode/dn-overview.html | 13 +-
.../src/main/resources/webapps/hddsDatanode/dn.js | 22 +-
.../resources/webapps/hddsDatanode}/index.html | 14 +-
.../main/resources/webapps/hddsDatanode/main.html | 20 ++
.../ozone/container/common/ContainerTestUtils.java | 3 +-
.../container/common/TestBlockDeletingService.java | 10 +-
.../container/common/TestDatanodeStateMachine.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../common/report/TestReportPublisher.java | 3 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 16 +-
.../keyvalue/TestKeyValueContainerCheck.java | 19 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 2 +-
.../ozoneimpl/TestContainerScrubberMetrics.java | 5 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +-
hadoop-hdds/docs/content/design/ofs.md | 143 ++++++++-
hadoop-hdds/docs/content/interface/OzoneFS.md | 58 +++-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 52 ++-
.../docs/content/security/SecuityWithRanger.zh.md | 35 +++
hadoop-hdds/docs/content/security/SecuringS3.md | 6 +-
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 50 +++
hadoop-hdds/docs/pom.xml | 2 +-
.../themes/ozonedoc/layouts/partials/footer.html | 2 +-
.../themes/ozonedoc/static/js/jquery-3.4.1.min.js | 2 -
.../themes/ozonedoc/static/js/jquery-3.5.1.min.js | 2 +
.../hdds/conf/DatanodeRatisServerConfig.java | 36 ++-
.../hadoop/hdds/server/http/BaseHttpServer.java | 10 +-
.../apache/hadoop/hdds/server/http/HttpConfig.java | 6 +-
.../hadoop/hdds/server/http/HttpServer2.java | 7 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 1 +
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 1 +
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 25 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 16 +
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 2 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 8 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 5 +
.../resources/webapps/static/angular-1.6.4.min.js | 332 -------------------
.../resources/webapps/static/angular-1.7.9.min.js | 350 +++++++++++++++++++++
.../webapps/static/angular-route-1.6.4.min.js | 17 -
.../webapps/static/angular-route-1.7.9.min.js | 17 +
.../resources/webapps/static/jquery-3.4.1.min.js | 2 -
.../resources/webapps/static/jquery-3.5.1.min.js | 2 +
.../hadoop/hdds/utils/TestMetadataStore.java | 48 +--
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 3 -
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 224 +++++++++++++
hadoop-hdds/interface-client/pom.xml | 2 +-
.../interface-client/src/main/proto/hdds.proto | 19 +-
.../interface-client/src/main/proto/proto.lock | 42 ++-
hadoop-hdds/pom.xml | 32 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 3 +-
.../hdds/scm/container/ContainerReportHandler.java | 9 +-
.../IncrementalContainerReportHandler.java | 13 +-
.../hdds/scm/container/ReplicationManager.java | 27 +-
.../hdds/scm/container/SCMContainerManager.java | 7 +-
.../hadoop/hdds/scm/metadata/PipelineIDCodec.java | 38 ++-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 69 ++--
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 49 ++-
.../hdds/scm/pipeline/SCMPipelineManager.java | 54 +++-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 5 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 5 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +-
.../server/StorageContainerManagerHttpServer.java | 6 +-
.../src/main/resources/webapps/scm/index.html | 6 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../TestIncrementalContainerReportHandler.java | 36 ++-
.../hdds/scm/container/TestReplicationManager.java | 4 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 144 +++++++++
.../hdds/scm/node/TestNodeReportHandler.java | 5 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 2 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 115 +++++++
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 7 +-
.../hdds/scm/cli/ContainerOperationClient.java | 4 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 21 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 13 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 3 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 32 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 22 ++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 20 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 9 +-
.../hadoop/ozone/om/exceptions/OMException.java | 2 +-
.../ozone/om/exceptions/OMReplayException.java | 50 ---
.../ozone/om/ha/OMFailoverProxyProvider.java | 2 +-
.../helpers/OmDeleteKeys.java} | 31 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 5 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 30 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 2 +-
.../hadoop/ozone/protocolPB/OzonePBHelper.java | 60 +++-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 12 +-
.../dev-support/checks/_mvn_unit_report.sh | 14 +-
hadoop-ozone/dev-support/checks/integration.sh | 25 +-
.../checks/{integration.sh => junit.sh} | 49 ++-
hadoop-ozone/dev-support/checks/unit.sh | 24 +-
hadoop-ozone/dist/pom.xml | 132 +-------
.../src/main/compose/ozone-csi/docker-compose.yaml | 1 +
.../dist/src/main/compose/ozone-csi/docker-config | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop31/docker-config | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 1 +
.../main/compose/ozone-mr/hadoop32/docker-config | 2 +-
.../compose/ozone-om-ha-s3/docker-compose.yaml | 1 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 1 +
.../src/main/compose/ozone-topology/docker-config | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 1 +
.../dist/src/main/compose/ozone/docker-config | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 1 +
.../src/main/compose/ozoneblockade/docker-config | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 2 +-
.../main/compose/ozonescripts/docker-compose.yaml | 1 +
.../src/main/compose/ozonescripts/docker-config | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 2 +-
.../compose/ozonesecure-om-ha/docker-compose.yaml | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 1 +
.../definitions/ozone/definitions/onenode.yaml} | 17 +-
.../main/k8s/examples/getting-started/Flekszible | 1 +
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../examples/regenerate-all.sh} | 17 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 6 +-
.../dist/src/shell/hdds/hadoop-functions.sh | 20 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 12 +-
.../src/test/compose/docker-compose.yaml | 1 +
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 72 ++++-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 158 +++++++++-
.../hadoop/fs/ozone/contract/OzoneContract.java | 15 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 15 +-
.../TestContainerStateManagerIntegration.java | 14 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 20 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 9 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 15 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 15 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 15 +-
.../rpc/TestCloseContainerHandlingByClient.java | 1 +
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 15 +-
.../rpc/TestContainerReplicationEndToEnd.java | 22 +-
.../TestContainerStateMachineFailureOnRead.java | 25 +-
.../rpc/TestContainerStateMachineFailures.java | 15 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 25 +-
.../client/rpc/TestFailureHandlingByClient.java | 15 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 15 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 15 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 5 -
.../client/rpc/TestValidateBCSIDOnRestart.java | 15 +-
.../ozone/client/rpc/TestWatchForCommit.java | 15 +-
.../hadoop/ozone/freon/TestDataValidate.java | 15 +-
.../freon/TestDataValidateWithDummyContainers.java | 4 -
.../ozone/freon/TestFreonWithDatanodeRestart.java | 15 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 32 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 15 +-
.../ozone/om/TestContainerReportWithKeys.java | 10 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 -
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 21 --
.../ozone/om/TestOzoneManagerHAWithData.java | 7 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 3 -
.../apache/hadoop/ozone/recon/TestReconTasks.java | 17 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../src/test/resources/ozone-site.xml | 2 +-
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../src/main/proto/OmClientProtocol.proto | 21 +-
.../interface-client/src/main/proto/Security.proto | 3 +-
.../interface-client/src/main/proto/proto.lock | 62 ++--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 4 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 24 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 8 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 36 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 89 +-----
.../om/request/bucket/OMBucketCreateRequest.java | 26 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 26 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 12 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 11 -
.../om/request/file/OMDirectoryCreateRequest.java | 47 +--
.../ozone/om/request/file/OMFileCreateRequest.java | 61 ++--
.../om/request/key/OMAllocateBlockRequest.java | 50 +--
.../ozone/om/request/key/OMKeyCommitRequest.java | 87 ++---
.../ozone/om/request/key/OMKeyCreateRequest.java | 60 ++--
.../ozone/om/request/key/OMKeyDeleteRequest.java | 36 +--
.../ozone/om/request/key/OMKeyPurgeRequest.java | 97 +-----
.../ozone/om/request/key/OMKeyRenameRequest.java | 130 +++-----
.../ozone/om/request/key/OMKeysDeleteRequest.java | 229 ++++++++------
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 24 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 6 -
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 6 -
.../om/request/key/acl/OMKeySetAclRequest.java | 6 -
.../request/key/acl/prefix/OMPrefixAclRequest.java | 30 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 6 -
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 6 -
.../key/acl/prefix/OMPrefixSetAclRequest.java | 6 -
.../S3InitiateMultipartUploadRequest.java | 5 -
.../multipart/S3MultipartUploadAbortRequest.java | 20 --
.../S3MultipartUploadCommitPartRequest.java | 34 +-
.../S3MultipartUploadCompleteRequest.java | 82 +----
.../security/OMCancelDelegationTokenRequest.java | 4 +-
.../security/OMGetDelegationTokenRequest.java | 6 +-
.../security/OMRenewDelegationTokenRequest.java | 5 +-
.../om/request/volume/OMVolumeCreateRequest.java | 23 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 -
.../om/request/volume/OMVolumeSetOwnerRequest.java | 57 +---
.../om/request/volume/OMVolumeSetQuotaRequest.java | 20 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 52 +--
.../request/volume/acl/OMVolumeAddAclRequest.java | 6 -
.../volume/acl/OMVolumeRemoveAclRequest.java | 6 -
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 -
.../hadoop/ozone/om/response/OMClientResponse.java | 2 +-
.../om/response/bucket/OMBucketCreateResponse.java | 2 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 2 +-
.../bucket/OMBucketSetPropertyResponse.java | 2 +-
.../response/bucket/acl/OMBucketAclResponse.java | 2 +-
.../response/file/OMDirectoryCreateResponse.java | 44 +--
.../om/response/file/OMFileCreateResponse.java | 4 +-
.../om/response/key/OMAllocateBlockResponse.java | 2 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 24 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 2 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 46 ++-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 11 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 59 +---
.../om/response/key/OMKeysDeleteResponse.java | 100 +++---
.../om/response/key/acl/OMKeyAclResponse.java | 2 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 2 +-
.../S3InitiateMultipartUploadResponse.java | 2 +-
.../multipart/S3MultipartUploadAbortResponse.java | 2 +-
.../S3MultipartUploadCommitPartResponse.java | 40 +--
.../S3MultipartUploadCompleteResponse.java | 48 +--
.../om/response/volume/OMVolumeAclOpResponse.java | 2 +-
.../om/response/volume/OMVolumeCreateResponse.java | 2 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 4 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 4 +-
.../main/resources/webapps/ozoneManager/index.html | 6 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 6 +-
.../request/bucket/TestOMBucketCreateRequest.java | 28 --
.../request/bucket/TestOMBucketDeleteRequest.java | 42 ---
.../bucket/TestOMBucketSetPropertyRequest.java | 29 --
.../om/request/file/TestOMFileCreateRequest.java | 29 --
.../ozone/om/request/key/TestOMKeyAclRequest.java | 9 +-
.../om/request/key/TestOMKeyCommitRequest.java | 79 -----
.../om/request/key/TestOMKeyCreateRequest.java | 42 ---
.../om/request/key/TestOMKeyDeleteRequest.java | 37 ---
.../key/TestOMKeyPurgeRequestAndResponse.java | 117 -------
.../om/request/key/TestOMKeyRenameRequest.java | 102 ------
.../om/request/key/TestOMKeysDeleteRequest.java | 155 +++++++++
.../om/request/key/TestOMPrefixAclRequest.java | 11 +-
.../security/TestOMGetDelegationTokenRequest.java | 3 +-
.../request/volume/TestOMVolumeCreateRequest.java | 26 --
.../request/volume/TestOMVolumeDeleteRequest.java | 38 ---
.../volume/TestOMVolumeSetOwnerRequest.java | 31 --
.../volume/TestOMVolumeSetQuotaRequest.java | 30 --
.../volume/acl/TestOMVolumeAddAclRequest.java | 33 --
.../volume/acl/TestOMVolumeRemoveAclRequest.java | 44 ---
.../volume/acl/TestOMVolumeSetAclRequest.java | 35 ---
.../file/TestOMDirectoryCreateResponse.java | 5 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 125 ++++++++
.../security/TestOMGetDelegationTokenResponse.java | 2 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 47 +++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 ++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 12 +
hadoop-ozone/pom.xml | 32 +-
.../ozone/recon/codegen/JooqCodeGenerator.java | 1 +
.../ozone/recon/codegen/ReconSqlDbConfig.java | 7 -
hadoop-ozone/recon/pom.xml | 2 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 -
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 124 +++++++-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 2 +
.../ozone/recon/api/types/DatanodeMetadata.java | 28 ++
.../api/types/UnhealthyContainerMetadata.java | 119 +++++++
.../api/types/UnhealthyContainersResponse.java | 98 ++++++
.../api/types/UnhealthyContainersSummary.java | 27 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 4 +-
.../recon/persistence/ContainerSchemaManager.java | 50 ++-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 4 +-
.../ReconIncrementalContainerReportHandler.java | 12 +-
.../recon/spi/ContainerDBServiceProvider.java | 8 -
.../spi/impl/ContainerDBServiceProviderImpl.java | 6 -
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 26 +-
.../webapps/recon/ozone-recon-web/api/db.json | 48 ++-
.../src/components/multiSelect/multiSelect.tsx | 5 +-
.../src/views/datanodes/datanodes.less | 14 +
.../src/views/datanodes/datanodes.tsx | 100 +++++-
.../ozone/recon/api/TestContainerEndpoint.java | 224 ++++++++++++-
.../hadoop/ozone/recon/api/TestEndpoints.java | 10 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 3 +-
...TestReconIncrementalContainerReportHandler.java | 30 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 4 +-
.../s3gateway/src/main/resources/browser.html | 2 +-
.../src/main/resources/webapps/static/index.html | 2 +-
.../src/main/resources/webapps/static/s3g.js | 4 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 4 +-
.../ozone/admin/om/GetServiceRolesSubcommand.java | 28 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 29 ++
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 6 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +-
pom.xml | 2 +-
362 files changed, 5122 insertions(+), 3811 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index e0cdeb1,a473948..8ed5ab6
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@@ -342,11 -362,6 +362,14 @@@ public final class OzoneConsts
public static final String TRANSACTION_INFO_KEY = "#TRANSACTIONINFO";
public static final String TRANSACTION_INFO_SPLIT_KEY = "#";
+ public static final String CONTAINER_DB_TYPE_ROCKSDB = "RocksDB";
+ public static final String CONTAINER_DB_TYPE_LEVELDB = "LevelDB";
++
+ // SCM HA
+ public static final String SCM_SERVICE_ID_DEFAULT = "scmServiceIdDefault";
+
+ // SCM Ratis snapshot file to store the last applied index
+ public static final String SCM_RATIS_SNAPSHOT_INDEX = "scmRatisSnapshotIndex";
+
+ public static final String SCM_RATIS_SNAPSHOT_TERM = "scmRatisSnapshotTerm";
}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index 4f8c1d6,e0ba53c..ad09ca2
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@@ -47,8 -45,9 +47,9 @@@ import org.apache.hadoop.hdds.scm.metad
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl;
import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
-import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org