You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2020/06/27 15:34:47 UTC
[hadoop-ozone] branch HDDS-2823 updated (3e7c427 -> 82c30a4)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a change to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from 3e7c427 Merge branch 'master' into HDDS-2823
add 3c43069 HDDS-3795. No coverage reported for Ozone FS (#1071)
add e663992 HDDS-3785. Update topology.aware.read parameter in ozone-topology compose config (#1070)
add aa04ac0 HDDS-3642. Stop/Pause Background services while replacing OM DB with checkpoint from Leader (#1002)
add 23034fb Hdds 3772. Add LOG to S3ErrorTable for easier problem locating. (#1054)
add 5792c62 HDDS-3769. hadoop-hdds interface-client fail to build with JDK11. (#1064)
add 57a8388 HDDS-3784. Use matrix build for integration test (#1063)
add 2505b5c HDDS-3787. Make CSI command configurable. (#1067)
add 348f7c6 HDDS-3781. Update k8s Daemonset/Deployment apiVersions to the newer one (#1056)
add e25c7a6 HDDS-3350. Ozone Retry Policy Improvements. (#1005)
add c5b0ba6 HDDS-3812. Disable netty resource leak detector in datanode. (#1080)
add 20bb5b5 HDDS-2840. Implement ofs://: mkdir (#415)
add 79c0c1c HDDS-2665. Merge master to HDDS-2665-ofs branch (#511)
add 3d39723 Revert "HDDS-2665. Merge master to HDDS-2665-ofs branch (#511)"
add ec9986b Merge branch 'master' into HDDS-2665
add 87288a2 HDDS-2665. Addendum fix for merge master due to HDDS-2188 revert. Contributed by Siyao Meng.
add 3e4782d HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
add 7c35d76 HDDS-2928. Implement ofs://: listStatus (#547)
add 0610ce3 HDDS-3073. Implement ofs://: Fix listStatus continuation (#606)
add c410c4d HDDS-2929. Implement ofs://: temp directory mount (#610)
add 5616dc3 HDDS-2945. Implement ofs://: Add robot tests for mkdir. (#703)
add 3c5ec6a Merge branch 'master' into HDDS-2665-ofs
add 3eaec50 HDDS-3279. Rebase OFS branch (#731)
add 8356a4a HDDS-3390. Adapt OFSPath to master branch changes (#847)
add dd99223 Merge remote-tracking branch 'asf/master' into HDDS-2665-ofs
add 9ed2e15 HDDS-3390. Rebase OFS branch - 2. Adapt OFS classes to HDDS-3101 (#822)
add 0773a4a HDDS-3494. Implement ofs://: Support volume and bucket deletion (#906)
add 28540cb HDDS-3574. Implement ofs://: Override getTrashRoot (#941)
add d6d31c6 HDDS-2969. Implement ofs://: Add contract test (#865)
add a00dc18 Merge remote-tracking branch 'asf/master' into HDDS-2665-ofs
add b9dac41 HDDS-3709. Rebase OFS branch - 3. Adapt to HDDS-3501 (#1015)
add f229222 Merge remote-tracking branch 'origin/master' into HDDS-2665-ofs
add f64315f HDDS-3767. [OFS] Address merge conflicts after HDDS-3627
add 2eb3181 Merge remote-tracking branch 'origin/master' into HDDS-2665-ofs
add 75de083 Merge branch 'HDDS-2665-ofs'
add 41fade2 HDDS-3262. Fix TestOzoneRpcClientWithRatis.java. (#1061)
add 60d895f HDDS-3797. Fix the mismatched dependency versions in submodule hadoop-ozone-filesystem-hadoop2 (#1073)
add ec8ee77 HDDS-3793. Use Hadoop 2.7.3 for ozone-mr/hadoop27 acceptance tests (#1074)
add 73b87fd HDDS-3776. Upgrading RocksDB version to avoid java heap issue (#1077)
add 52ba5e8 HDDS-3818. Schedule daily 2 builds from master branch build (#1086)
add 4c3f9d5 HDDS-3796. Allow running coverage locally (#1072)
add 5ebb065 HDDS-3802. Incorrect data returned by reading a FILE_PER_CHUNK block. (#1079)
add d471e04 HDDS-3743. Avoid NetUtils#normalize when get DatanodeDetails from proto (#1044)
add 8b2f9f4 HDDS-3794. Topology Aware read does not work correctly in XceiverClientGrpc (#1078)
add 8f7852d HDDS-3792. Separate client proto files of Ozone to separated subprojects (#1100)
add 1326aaa HDDS-3428. Fix failing test cases in TestOzoneRpcClientWithRatis. (#1095)
add 98a5bbc HDDS-3822. Eliminate duplicated GitHub Actions workflow (#1090)
add c349bb9 HDDS-3842. Cache firstNonFullIndex in IncrementalChunkBuffer. (#1103)
add 8b05c91 HDDS-3823. Hadoop3 artifact should depend on the ozonefs-shaded (#1094)
add 53549ee HDDS-2800. tools/_index.md translation (#780)
add 3c30f04 HDDS-3094. Save each output of smoketest executed multiple times (#1091)
add 755170d HDDS-3826. Split Ozone FS acceptance tests (#1093)
add c37a257 HDDS-3825. Show rule status of SCM safemode (#1102)
add 7ff5b5b HDDS-3828. Configuration parsing of ozone insight should be based on fields (#1105)
add a42aa72 HDDS-3815. Avoid buffer copy in ContainerCommandRequestProto. (#1085)
add f8fcc47 HDDS-3848. Add ratis.thirdparty.version in main pom.xml (#1108)
add 69dd3ff HDDS-3850. Update the admin document to let user know how to show the status of all rules. (#1109)
add a5556d3 HDDS-3286. BasicOzoneFileSystem support batchDelete. (#814)
add 3ca7f5c HDDS-3745. Improve OM and SCM performance with 64% by avoid collect datanode information to s3g (#1031)
add f265723 HDDS-3847. Change OMNotLeaderException logging to DEBUG (#1118)
add 7185a84 HDDS-3780. Replace the imagePullPolicy from always to IfNotPresent (#1055)
add 74603c0 HDDS-3836. Modify ContainerPlacementPolicyFactory JavaDoc (#1097)
add fb3902f HDDS-3512. s3g multi-part-upload saved incorrect content using streaming (#1092)
add 07f7131 HDDS-3263. Fix TestCloseContainerByPipeline.java. (#1119)
add d30ff3e HDDS-3430. Enable TestWatchForCommit test cases. (#1114)
add d3b4642 HDDS-3857. Datanode in compose/ozonescripts can't be started (#1116)
add a89ff6d HDDS-3779. Add csi interface documents to show how to use ozone csi (#1059)
add d163541 HDDS-3819. OzoneManager#listVolumeByUser ignores userName parameter when ACL is enabled (#1087)
add ecfbd76 HDDS-3821. Disable Ozone SPNEGO should not fall back to hadoop.http.a… (#1101)
add ac9387c Revert "HDDS-3263. Fix TestCloseContainerByPipeline.java. (#1119)" (#1126)
add 5e7b2b6 HDDS-3773. Add OMDBDefinition to define structure of om.db. (#1076)
add fe015bd HDDS-3704. Update all the documentation to use ozonefs-hadoop2/3 instead of legacy/current (#1099)
add 8235366 HDDS-3858. Remove support to start Ozone and HDFS datanodes in the same JVM (#1117)
add a27c4b3 HDDS-3264. Fix TestCSMMetrics.java. (#1120)
add 5d14b1c HDDS-3866. Support multi-part-upload with Freon S3 key generator (#1125)
add 025fc54 HDDS-3018. Fix TestContainerStateMachineFailures.java (#556)
add 93b1f63 HDDS-3479. Use SCMMetadataStore high level abstraction instead of DBS… (#997)
add e498a80 HDDS-3854. Fix error return value of KeyValueBlockIterator#hasNext (#1111)
add a790c6f HDDS-2765. security/SecureOzone.md translation (#870)
add 38a4786 HDDS-3871. Add resource core-site during loading of ozoneconfiguration. (#1130)
add 9779b0c HDDS-426. Add field modificationTime for Volume and Bucket (#164)
add 90c17ca HDDS-3615. Call cleanup on tables only when double buffer has transactions related to tables. (#943)
add 0f2a118 HDDS-3757. Add test coverage of the acceptance tests to overall test coverage (#1050)
add d522884 HDDS-3872. Reduce output of coverage.sh (#1132)
new 82c30a4 Merge branch 'master' into HDDS-2823
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 183 ++---
.github/workflows/pr.yml | 270 ------
.../apache/hadoop/hdds/conf/RatisClientConfig.java | 86 --
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 21 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 5 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 6 +
.../hdds/ratis/ContainerCommandRequestMessage.java | 6 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 214 +++--
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 183 +++++
.../hadoop/hdds/ratis}/conf/package-info.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 12 +-
.../protocol/StorageContainerLocationProtocol.java | 4 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 14 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../ozone/common/IncrementalChunkBuffer.java | 12 +-
.../common/src/main/resources/ozone-default.xml | 21 +-
.../apache/hadoop/hdds/scm/net/TestNetUtils.java | 1 -
.../server/ratis/ContainerStateMachine.java | 14 +-
.../transport/server/ratis/XceiverServerRatis.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 3 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 2 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 5 +-
.../keyvalue/impl/ChunkManagerFactory.java | 13 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 45 +-
.../common/impl/TestContainerPersistence.java | 7 +-
.../container/keyvalue/ChunkLayoutTestInfo.java | 12 +-
.../keyvalue/TestKeyValueBlockIterator.java | 3 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 8 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 7 +
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 70 --
.../docs/content/beyond/RunningWithHDFS.zh.md | 64 --
hadoop-hdds/docs/content/interface/CSI.md | 84 ++
hadoop-hdds/docs/content/interface/OzoneFS.md | 53 +-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 37 +-
.../docs/content/interface/pvtest-webui.png | Bin 0 -> 13830 bytes
hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 30 +-
.../docs/content/recipe/SparkOzoneFSK8S.zh.md | 36 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 20 +-
.../docs/content/security/SecureOzone.zh.md | 158 ++++
.../docs/content/security/SecuringOzoneHTTP.md | 141 ++++
hadoop-hdds/docs/content/tools/Admin.md | 2 +-
hadoop-hdds/docs/content/tools/_index.zh.md | 62 ++
...inerLocationProtocolClientSideTranslatorPB.java | 24 +
.../hadoop/hdds/server/http/BaseHttpServer.java | 4 +-
.../hadoop/hdds/server/http/HttpServer2.java | 22 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 11 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 17 +-
.../interface-admin/src/main/proto/proto.lock | 48 ++
hadoop-hdds/interface-client/pom.xml | 6 +-
.../ContainerPlacementPolicyFactory.java | 2 +-
...StoreRDBImpl.java => SCMMetadataStoreImpl.java} | 6 +-
...inerLocationProtocolServerSideTranslatorPB.java | 28 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 8 +-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 8 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 9 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 9 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 15 +
.../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 5 +
.../hdds/scm/server/SCMClientProtocolServer.java | 7 +
.../hdds/scm/server/StorageContainerManager.java | 14 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../container/TestCloseContainerEventHandler.java | 28 +-
.../scm/container/TestSCMContainerManager.java | 13 +-
.../hdds/scm/node/TestContainerPlacement.java | 20 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 36 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 121 ++-
.../TestOneReplicaPipelineSafeModeRule.java | 11 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 25 +-
hadoop-hdds/test-utils/pom.xml | 6 +
.../java/org/apache/hadoop/test/JacocoServer.java | 114 +++
.../hdds/scm/cli/ContainerOperationClient.java | 7 +
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 15 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 70 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 69 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 6 -
hadoop-ozone/common/pom.xml | 29 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 33 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 33 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 10 +
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 58 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 29 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 16 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 17 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 5 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
.../dev-support/checks/coverage.sh | 16 +-
hadoop-ozone/dist/pom.xml | 53 ++
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 +
.../src/main/compose/ozone-csi/docker-compose.yaml | 6 +
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 18 -
.../main/compose/ozone-hdfs/docker-compose.yaml | 70 --
.../dist/src/main/compose/ozone-hdfs/docker-config | 35 -
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +-
.../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 +
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 6 +
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 +
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozone-om-ha-s3/.env | 1 +
.../compose/ozone-om-ha-s3/docker-compose.yaml | 8 +
.../dist/src/main/compose/ozone-topology/.env | 1 +
.../compose/ozone-topology/docker-compose.yaml | 14 +
.../src/main/compose/ozone-topology/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 5 +
.../dist/src/main/compose/ozone/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +-
.../src/main/compose/ozonescripts/docker-config | 9 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozonesecure-om-ha/.env | 1 +
.../compose/ozonesecure-om-ha/docker-compose.yaml | 13 +-
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 5 +
.../src/main/compose/ozonesecure/docker-config | 1 +
.../dist/src/main/compose/ozonesecure/test.sh | 9 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 18 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 15 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/definitions/ozone-csi/csi-node.yaml | 4 +-
.../k8s/definitions/prometheus/deployment.yaml | 2 +-
.../examples/ozone-dev/csi/csi-node-daemonset.yaml | 4 +-
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 2 +-
.../examples/ozone-dev/prometheus-deployment.yaml | 2 +-
.../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 4 +-
.../ozone/csi/csi-provisioner-deployment.yaml | 2 +-
.../dist/src/main/smoketest/commonlib.robot | 6 +-
hadoop-ozone/dist/src/main/smoketest/lib/fs.robot | 50 ++
.../dist/src/main/smoketest/lib/fs_tests.robot | 63 ++
.../{s3/bucketlist.robot => lib/string.robot} | 30 +-
.../bucketcreate.robot => lib/string_tests.robot} | 43 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 217 ++---
.../dist/src/main/smoketest/ozonefs/setup.robot | 60 ++
hadoop-ozone/dist/src/shell/ozone/ozone | 4 +-
.../ozone/insight/ConfigurationSubCommand.java | 31 +-
.../ozone/insight/TestConfigurationSubCommand.java | 100 +++
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 874 ++++++++++++++++++++
.../ITestRootedOzoneContractCreate.java} | 12 +-
.../ITestRootedOzoneContractDelete.java} | 12 +-
.../ITestRootedOzoneContractDistCp.java} | 14 +-
.../ITestRootedOzoneContractGetFileStatus.java} | 23 +-
.../ITestRootedOzoneContractMkdir.java} | 12 +-
.../ITestRootedOzoneContractOpen.java} | 12 +-
.../ITestRootedOzoneContractRename.java} | 12 +-
.../ITestRootedOzoneContractRootDir.java} | 22 +-
.../ITestRootedOzoneContractSeek.java} | 12 +-
.../RootedOzoneContract.java} | 32 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 6 -
.../rpc/TestBlockOutputStreamWithFailures.java | 5 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 5 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 8 +-
.../TestContainerStateMachineFailureOnRead.java | 6 +-
.../rpc/TestContainerStateMachineFailures.java | 422 +++++-----
.../client/rpc/TestDeleteWithSlowFollower.java | 8 +-
.../client/rpc/TestFailureHandlingByClient.java | 8 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +-
.../ozone/client/rpc/TestKeyInputStream.java | 82 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 8 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 17 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 4 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 258 ++++++
.../ozone/client/rpc/TestWatchForCommit.java | 170 ++--
.../apache/hadoop/ozone/container/TestHelper.java | 41 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 -
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 7 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 41 +
.../ozone/om/TestOzoneManagerListVolumes.java | 62 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 25 -
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 198 +++++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 89 ++
.../src/test/resources/log4j.properties | 3 +
.../dev-support/findbugsExcludeFile.xml | 1 -
.../interface-client}/pom.xml | 17 +-
.../src/main/proto/OmClientProtocol.proto} | 15 +
.../src/main/proto/Security.proto | 0
.../src/main/proto/proto.lock | 59 +-
hadoop-ozone/ozone-manager/pom.xml | 7 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 20 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 23 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 99 ++-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 161 ++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 96 ++-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMClientRequest.java | 34 +
.../om/request/bucket/OMBucketCreateRequest.java | 6 +-
...DeleteRequest.java => OMKeysDeleteRequest.java} | 194 ++---
.../om/request/volume/OMVolumeCreateRequest.java | 8 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 22 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 17 +
.../hadoop/ozone/om/response/CleanupTableInfo.java | 21 +-
.../om/response/bucket/OMBucketCreateResponse.java | 4 +
.../om/response/bucket/OMBucketDeleteResponse.java | 4 +
.../bucket/OMBucketSetPropertyResponse.java | 4 +
.../response/bucket/acl/OMBucketAclResponse.java | 5 +-
.../response/file/OMDirectoryCreateResponse.java | 5 +
.../om/response/key/OMAllocateBlockResponse.java | 4 +
.../ozone/om/response/key/OMKeyCommitResponse.java | 6 +
.../ozone/om/response/key/OMKeyCreateResponse.java | 4 +
.../ozone/om/response/key/OMKeyDeleteResponse.java | 6 +
.../ozone/om/response/key/OMKeyPurgeResponse.java | 6 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 4 +
.../om/response/key/OMKeysDeleteResponse.java | 133 +++
.../om/response/key/OMTrashRecoverResponse.java | 5 +
.../om/response/key/acl/OMKeyAclResponse.java | 4 +
.../key/acl/prefix/OMPrefixAclResponse.java | 4 +
.../S3InitiateMultipartUploadResponse.java | 6 +-
.../multipart/S3MultipartUploadAbortResponse.java | 7 +
.../S3MultipartUploadCommitPartResponse.java | 7 +
.../S3MultipartUploadCompleteResponse.java | 9 +
.../response/s3/security/S3GetSecretResponse.java | 5 +-
.../security/OMCancelDelegationTokenResponse.java | 5 +-
.../security/OMGetDelegationTokenResponse.java | 6 +-
.../security/OMRenewDelegationTokenResponse.java | 6 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 4 +
.../om/response/volume/OMVolumeCreateResponse.java | 4 +
.../om/response/volume/OMVolumeDeleteResponse.java | 4 +
.../response/volume/OMVolumeSetOwnerResponse.java | 5 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 5 +
...tOzoneManagerDoubleBufferWithDummyResponse.java | 4 +-
.../ozone/om/request/TestOMRequestUtils.java | 4 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +
.../request/volume/TestOMVolumeCreateRequest.java | 7 +
.../volume/TestOMVolumeSetOwnerRequest.java | 7 +
.../volume/TestOMVolumeSetQuotaRequest.java | 7 +
.../ozone/om/response/TestCleanupTableInfo.java | 69 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 24 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 63 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 912 +++++++++++++++++++++
...System.java => BasicRootedOzoneFileSystem.java} | 320 +++++---
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 311 +++++++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 +
...Impl.java => RootedOzoneClientAdapterImpl.java} | 22 +-
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 154 ++++
hadoop-ozone/ozonefs-hadoop2/pom.xml | 12 +
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 5 +
...eFileSystem.java => RootedOzoneFileSystem.java} | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 17 +-
...eFileSystem.java => RootedOzoneFileSystem.java} | 32 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java} | 32 +-
hadoop-ozone/pom.xml | 6 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 6 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 +
.../hadoop/ozone/client/ObjectStoreStub.java | 6 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 10 +-
.../{RDBParser.java => DBDefinitionFactory.java} | 43 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 11 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 3 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 72 +-
.../ozone/genesis/BenchmarkChunkManager.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 13 +-
.../ozone/shell/volume/UpdateVolumeHandler.java | 4 +-
pom.xml | 8 +-
262 files changed, 7392 insertions(+), 2421 deletions(-)
delete mode 100644 .github/workflows/pr.yml
delete mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/conf/RatisClientConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java
rename hadoop-hdds/{client/src/main/java/org/apache/hadoop/hdds => common/src/main/java/org/apache/hadoop/hdds/ratis}/conf/package-info.java (95%)
delete mode 100644 hadoop-hdds/docs/content/beyond/RunningWithHDFS.md
delete mode 100644 hadoop-hdds/docs/content/beyond/RunningWithHDFS.zh.md
create mode 100644 hadoop-hdds/docs/content/interface/CSI.md
create mode 100644 hadoop-hdds/docs/content/interface/pvtest-webui.png
create mode 100644 hadoop-hdds/docs/content/security/SecureOzone.zh.md
create mode 100644 hadoop-hdds/docs/content/security/SecuringOzoneHTTP.md
create mode 100644 hadoop-hdds/docs/content/tools/_index.zh.md
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/{SCMMetadataStoreRDBImpl.java => SCMMetadataStoreImpl.java} (96%)
create mode 100644 hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/JacocoServer.java
rename .github/coverage-report.sh => hadoop-ozone/dev-support/checks/coverage.sh (80%)
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
create mode 100644 hadoop-ozone/dist/src/main/smoketest/lib/fs.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/lib/fs_tests.robot
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketlist.robot => lib/string.robot} (61%)
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => lib/string_tests.robot} (50%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot
create mode 100644 hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractCreate.java => rooted/ITestRootedOzoneContractCreate.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractDelete.java => rooted/ITestRootedOzoneContractDelete.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractDistCp.java => rooted/ITestRootedOzoneContractDistCp.java} (78%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractGetFileStatus.java => rooted/ITestRootedOzoneContractGetFileStatus.java} (72%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractMkdir.java => rooted/ITestRootedOzoneContractMkdir.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractOpen.java => rooted/ITestRootedOzoneContractOpen.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractRename.java => rooted/ITestRootedOzoneContractRename.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractRootDir.java => rooted/ITestRootedOzoneContractRootDir.java} (72%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{ITestOzoneContractSeek.java => rooted/ITestRootedOzoneContractSeek.java} (81%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/{OzoneContract.java => rooted/RootedOzoneContract.java} (82%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientGrpc.java
copy hadoop-ozone/{common => interface-client}/dev-support/findbugsExcludeFile.xml (96%)
copy {hadoop-hdds/interface-admin => hadoop-ozone/interface-client}/pom.xml (84%)
rename hadoop-ozone/{common/src/main/proto/OzoneManagerProtocol.proto => interface-client/src/main/proto/OmClientProtocol.proto} (98%)
rename hadoop-ozone/{common => interface-client}/src/main/proto/Security.proto (100%)
rename hadoop-ozone/{common => interface-client}/src/main/proto/proto.lock (98%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/{OMKeyDeleteRequest.java => OMKeysDeleteRequest.java} (52%)
copy hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java (66%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
copy hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/{BasicOzoneFileSystem.java => BasicRootedOzoneFileSystem.java} (71%)
create mode 100644 hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
copy hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/{OzoneClientAdapterImpl.java => RootedOzoneClientAdapterImpl.java} (72%)
create mode 100644 hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
copy hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/{OzoneFileSystem.java => RootedOzoneFileSystem.java} (86%)
copy hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/{OzoneFileSystem.java => RootedOzoneFileSystem.java} (83%)
copy hadoop-ozone/{ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java => ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java} (83%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/{RDBParser.java => DBDefinitionFactory.java} (55%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merge branch 'master' into HDDS-2823
Posted by na...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 82c30a4d26be4a0be4c3e3e071f4add41c9c324e
Merge: 3e7c427 d522884
Author: Nandakumar <na...@apache.org>
AuthorDate: Sat Jun 27 20:14:12 2020 +0530
Merge branch 'master' into HDDS-2823
.github/workflows/post-commit.yml | 183 ++---
.github/workflows/pr.yml | 270 ------
.../apache/hadoop/hdds/conf/RatisClientConfig.java | 86 --
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 21 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 5 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 6 +
.../hdds/ratis/ContainerCommandRequestMessage.java | 6 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 214 +++--
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 183 +++++
.../hadoop/hdds/ratis}/conf/package-info.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 12 +-
.../protocol/StorageContainerLocationProtocol.java | 4 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 14 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../ozone/common/IncrementalChunkBuffer.java | 12 +-
.../common/src/main/resources/ozone-default.xml | 21 +-
.../apache/hadoop/hdds/scm/net/TestNetUtils.java | 1 -
.../server/ratis/ContainerStateMachine.java | 14 +-
.../transport/server/ratis/XceiverServerRatis.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 3 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 2 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 5 +-
.../keyvalue/impl/ChunkManagerFactory.java | 13 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 45 +-
.../common/impl/TestContainerPersistence.java | 7 +-
.../container/keyvalue/ChunkLayoutTestInfo.java | 12 +-
.../keyvalue/TestKeyValueBlockIterator.java | 3 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 8 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 7 +
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 70 --
.../docs/content/beyond/RunningWithHDFS.zh.md | 64 --
hadoop-hdds/docs/content/interface/CSI.md | 84 ++
hadoop-hdds/docs/content/interface/OzoneFS.md | 53 +-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 37 +-
.../docs/content/interface/pvtest-webui.png | Bin 0 -> 13830 bytes
hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 30 +-
.../docs/content/recipe/SparkOzoneFSK8S.zh.md | 36 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 20 +-
.../docs/content/security/SecureOzone.zh.md | 158 ++++
.../docs/content/security/SecuringOzoneHTTP.md | 141 ++++
hadoop-hdds/docs/content/tools/Admin.md | 2 +-
hadoop-hdds/docs/content/tools/_index.zh.md | 62 ++
...inerLocationProtocolClientSideTranslatorPB.java | 24 +
.../hadoop/hdds/server/http/BaseHttpServer.java | 4 +-
.../hadoop/hdds/server/http/HttpServer2.java | 22 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 11 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 17 +-
.../interface-admin/src/main/proto/proto.lock | 48 ++
hadoop-hdds/interface-client/pom.xml | 6 +-
.../ContainerPlacementPolicyFactory.java | 2 +-
...StoreRDBImpl.java => SCMMetadataStoreImpl.java} | 6 +-
...inerLocationProtocolServerSideTranslatorPB.java | 28 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 8 +-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 8 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 9 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 9 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 15 +
.../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 5 +
.../hdds/scm/server/SCMClientProtocolServer.java | 7 +
.../hdds/scm/server/StorageContainerManager.java | 14 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../container/TestCloseContainerEventHandler.java | 28 +-
.../scm/container/TestSCMContainerManager.java | 13 +-
.../hdds/scm/node/TestContainerPlacement.java | 20 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 36 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 121 ++-
.../TestOneReplicaPipelineSafeModeRule.java | 11 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 25 +-
hadoop-hdds/test-utils/pom.xml | 6 +
.../java/org/apache/hadoop/test/JacocoServer.java | 114 +++
.../hdds/scm/cli/ContainerOperationClient.java | 7 +
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 15 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 70 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 69 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 6 -
hadoop-ozone/common/pom.xml | 29 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 33 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 33 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 10 +
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 58 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 29 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 16 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 17 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 5 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
.../dev-support/checks/coverage.sh | 16 +-
hadoop-ozone/dist/pom.xml | 53 ++
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 +
.../src/main/compose/ozone-csi/docker-compose.yaml | 6 +
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 18 -
.../main/compose/ozone-hdfs/docker-compose.yaml | 70 --
.../dist/src/main/compose/ozone-hdfs/docker-config | 35 -
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../main/compose/ozone-mr/hadoop27/docker-config | 2 +-
.../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 +
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 6 +
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 +
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozone-om-ha-s3/.env | 1 +
.../compose/ozone-om-ha-s3/docker-compose.yaml | 8 +
.../dist/src/main/compose/ozone-topology/.env | 1 +
.../compose/ozone-topology/docker-compose.yaml | 14 +
.../src/main/compose/ozone-topology/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 5 +
.../dist/src/main/compose/ozone/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +-
.../src/main/compose/ozonescripts/docker-config | 9 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 4 +
.../dist/src/main/compose/ozonesecure-om-ha/.env | 1 +
.../compose/ozonesecure-om-ha/docker-compose.yaml | 13 +-
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 5 +
.../src/main/compose/ozonesecure/docker-config | 1 +
.../dist/src/main/compose/ozonesecure/test.sh | 9 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 18 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 15 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/definitions/ozone-csi/csi-node.yaml | 4 +-
.../k8s/definitions/prometheus/deployment.yaml | 2 +-
.../examples/ozone-dev/csi/csi-node-daemonset.yaml | 4 +-
.../ozone-dev/csi/csi-provisioner-deployment.yaml | 2 +-
.../examples/ozone-dev/prometheus-deployment.yaml | 2 +-
.../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 4 +-
.../ozone/csi/csi-provisioner-deployment.yaml | 2 +-
.../dist/src/main/smoketest/commonlib.robot | 6 +-
hadoop-ozone/dist/src/main/smoketest/lib/fs.robot | 50 ++
.../dist/src/main/smoketest/lib/fs_tests.robot | 63 ++
.../src/main/smoketest/lib/string.robot} | 28 +-
.../dist/src/main/smoketest/lib/string_tests.robot | 46 ++
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 217 ++---
.../dist/src/main/smoketest/ozonefs/setup.robot | 60 ++
hadoop-ozone/dist/src/shell/ozone/ozone | 4 +-
.../ozone/insight/ConfigurationSubCommand.java | 31 +-
.../ozone/insight/TestConfigurationSubCommand.java | 100 +++
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 874 ++++++++++++++++++++
.../rooted/ITestRootedOzoneContractCreate.java | 49 ++
.../rooted/ITestRootedOzoneContractDelete.java | 49 ++
.../rooted/ITestRootedOzoneContractDistCp.java | 51 ++
.../ITestRootedOzoneContractGetFileStatus.java | 55 ++
.../rooted/ITestRootedOzoneContractMkdir.java | 49 ++
.../rooted/ITestRootedOzoneContractOpen.java | 48 ++
.../rooted/ITestRootedOzoneContractRename.java | 50 ++
.../rooted/ITestRootedOzoneContractRootDir.java | 61 ++
.../rooted/ITestRootedOzoneContractSeek.java | 48 ++
.../ozone/contract/rooted/RootedOzoneContract.java | 124 +++
.../ozone/client/rpc/Test2WayCommitInRatis.java | 6 -
.../rpc/TestBlockOutputStreamWithFailures.java | 5 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 5 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 8 +-
.../TestContainerStateMachineFailureOnRead.java | 6 +-
.../rpc/TestContainerStateMachineFailures.java | 422 +++++-----
.../client/rpc/TestDeleteWithSlowFollower.java | 8 +-
.../client/rpc/TestFailureHandlingByClient.java | 8 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +-
.../ozone/client/rpc/TestKeyInputStream.java | 82 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 8 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 17 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 4 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 258 ++++++
.../ozone/client/rpc/TestWatchForCommit.java | 170 ++--
.../apache/hadoop/ozone/container/TestHelper.java | 41 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 -
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 7 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 41 +
.../ozone/om/TestOzoneManagerListVolumes.java | 62 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 25 -
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 198 +++++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 89 ++
.../src/test/resources/log4j.properties | 3 +
.../dev-support/findbugsExcludeFile.xml | 1 -
hadoop-ozone/interface-client/pom.xml | 77 ++
.../src/main/proto/OmClientProtocol.proto} | 15 +
.../src/main/proto/Security.proto | 0
.../src/main/proto/proto.lock | 59 +-
hadoop-ozone/ozone-manager/pom.xml | 7 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 20 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 23 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 99 ++-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 161 ++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 96 ++-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../hadoop/ozone/om/request/OMClientRequest.java | 34 +
.../om/request/bucket/OMBucketCreateRequest.java | 6 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 214 +++++
.../om/request/volume/OMVolumeCreateRequest.java | 8 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 22 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 17 +
.../ozone/om/response/CleanupTableInfo.java} | 40 +-
.../om/response/bucket/OMBucketCreateResponse.java | 4 +
.../om/response/bucket/OMBucketDeleteResponse.java | 4 +
.../bucket/OMBucketSetPropertyResponse.java | 4 +
.../response/bucket/acl/OMBucketAclResponse.java | 5 +-
.../response/file/OMDirectoryCreateResponse.java | 5 +
.../om/response/key/OMAllocateBlockResponse.java | 4 +
.../ozone/om/response/key/OMKeyCommitResponse.java | 6 +
.../ozone/om/response/key/OMKeyCreateResponse.java | 4 +
.../ozone/om/response/key/OMKeyDeleteResponse.java | 6 +
.../ozone/om/response/key/OMKeyPurgeResponse.java | 6 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 4 +
.../om/response/key/OMKeysDeleteResponse.java | 133 +++
.../om/response/key/OMTrashRecoverResponse.java | 5 +
.../om/response/key/acl/OMKeyAclResponse.java | 4 +
.../key/acl/prefix/OMPrefixAclResponse.java | 4 +
.../S3InitiateMultipartUploadResponse.java | 6 +-
.../multipart/S3MultipartUploadAbortResponse.java | 7 +
.../S3MultipartUploadCommitPartResponse.java | 7 +
.../S3MultipartUploadCompleteResponse.java | 9 +
.../response/s3/security/S3GetSecretResponse.java | 5 +-
.../security/OMCancelDelegationTokenResponse.java | 5 +-
.../security/OMGetDelegationTokenResponse.java | 6 +-
.../security/OMRenewDelegationTokenResponse.java | 6 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 4 +
.../om/response/volume/OMVolumeCreateResponse.java | 4 +
.../om/response/volume/OMVolumeDeleteResponse.java | 4 +
.../response/volume/OMVolumeSetOwnerResponse.java | 5 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 5 +
...tOzoneManagerDoubleBufferWithDummyResponse.java | 4 +-
.../ozone/om/request/TestOMRequestUtils.java | 4 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +
.../request/volume/TestOMVolumeCreateRequest.java | 7 +
.../volume/TestOMVolumeSetOwnerRequest.java | 7 +
.../volume/TestOMVolumeSetQuotaRequest.java | 7 +
.../ozone/om/response/TestCleanupTableInfo.java | 69 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 24 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 63 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 912 +++++++++++++++++++++
...System.java => BasicRootedOzoneFileSystem.java} | 320 +++++---
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 311 +++++++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 +
.../fs/ozone/RootedOzoneClientAdapterImpl.java | 61 ++
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 154 ++++
hadoop-ozone/ozonefs-hadoop2/pom.xml | 12 +
.../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 5 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 15 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 17 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 95 +++
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 95 +++
hadoop-ozone/pom.xml | 6 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 6 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 +
.../hadoop/ozone/client/ObjectStoreStub.java | 6 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 10 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 49 ++
.../org/apache/hadoop/ozone/debug/DBScanner.java | 11 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 3 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 72 +-
.../ozone/genesis/BenchmarkChunkManager.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 13 +-
.../ozone/shell/volume/UpdateVolumeHandler.java | 4 +-
pom.xml | 8 +-
262 files changed, 8358 insertions(+), 2158 deletions(-)
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 69a8013,73113d1..20d0480
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -1152,53 -1112,12 +1153,62 @@@ public final class StorageContainerMana
return this.clusterMap;
}
+ private static SCMRatisServer initializeRatisServer(
+ OzoneConfiguration conf, StorageContainerManager scm) throws IOException {
+ SCMNodeDetails scmNodeDetails = SCMNodeDetails
+ .initStandAlone(conf);
+ //TODO enable Ratis group
+ SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer(
+ conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class),
+ scm, scmNodeDetails, Collections.EMPTY_LIST,
+ SCMRatisServer.getSCMRatisDirectory(conf));
+ if (scmRatisServer != null) {
+ LOG.info("SCM Ratis server initialized at port {}",
+ scmRatisServer.getServerPort());
+ } // TODO error handling for scmRatisServer creation failure
+ return scmRatisServer;
+ }
+
+ @VisibleForTesting
+ public SCMRatisServer getScmRatisServer() {
+ return scmRatisServer;
+ }
+
+ public void setScmRatisServer(SCMRatisServer scmRatisServer) {
+ this.scmRatisServer = scmRatisServer;
+ }
+
+ @VisibleForTesting
+ public SCMRatisSnapshotInfo getSnapshotInfo() {
+ return scmRatisSnapshotInfo;
+ }
+
+ @VisibleForTesting
+ public long getRatisSnapshotIndex() {
+ return scmRatisSnapshotInfo.getIndex();
+ }
+
+ /**
+ * Save ratis snapshot to SCM meta store and local disk.
+ */
+ public TermIndex saveRatisSnapshot() throws IOException {
+ TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex();
+ if (scmMetadataStore != null) {
+ // Flush the SCM state to disk
+ scmMetadataStore.getStore().flush();
+ }
+
+ scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex);
+
+ return snapshotIndex;
+ }
++
+ /**
+ * Get the safe mode status of all rules.
+ *
+ * @return map of rule statuses.
+ */
+ public Map<String, Pair<Boolean, String>> getRuleStatus() {
+ return scmSafeModeManager.getRuleStatus();
+ }
}
diff --cc hadoop-ozone/interface-client/src/main/proto/proto.lock
index 01eedd0,7e41dfa..91ec7f0
--- a/hadoop-ozone/interface-client/src/main/proto/proto.lock
+++ b/hadoop-ozone/interface-client/src/main/proto/proto.lock
@@@ -1215,6 -1229,6 +1229,11 @@@
"id": 9,
"name": "updateID",
"type": "uint64"
++ },
++ {
++ "id": 10,
++ "name": "modificationTime",
++ "type": "uint64"
}
]
},
@@@ -1319,6 -1333,6 +1338,11 @@@
"id": 3,
"name": "quotaInBytes",
"type": "uint64"
++ },
++ {
++ "id": 4,
++ "name": "modificationTime",
++ "type": "uint64"
}
]
},
@@@ -1490,6 -1504,6 +1514,11 @@@
"id": 10,
"name": "updateID",
"type": "uint64"
++ },
++ {
++ "id": 11,
++ "name": "modificationTime",
++ "type": "uint64"
}
]
},
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org