You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2020/10/06 07:38:04 UTC
[hadoop-ozone] branch HDDS-1880-Decom updated (f43a370 -> 5ab169a)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from f43a370 HDDS-4300. Removed unneeded class DatanodeAdminNodeDetails (#1465)
add 34f3b91 HDDS-4232. Use single thread for KeyDeletingService. (#1415)
add 68642c2 HDDS-4023. Delete closed container after all blocks have been deleted. (#1338)
add 3ad1034 HDDS-4215. Update Freon doc in source tree. (#1403)
add 2420ee8 HDDS-4288. the icon of hadoop-ozone is bigger than ever (#1452)
add fdcc696 HDDS-4287: Exclude protobuff classes from ozone-filesystem-hadoop3 jars (#1455). Contributed by Uma Maheswara Rao G.
add d1ac423 HDDS-3751. Ozone sh client support bucket quota option. (#1412)
add be25991 HDDS-4292. Ozone Client not working with Hadoop Version < 3.2 (#1463)
add d6a1836 HDDS-4251. Update Ratis version to latest snapshot (#1462)
add 275653e HDDS-3869. Use different column families for datanode block and metadata (#1298)
add 819b455 HDDS-4302 : Shade the org.apache.common.lang3 package as this is coming from other hadoop packages as well. (#1469). Contributed by Uma Maheswara Rao G.
add 7216e3c HDDS-3966. Enable TestOMRatisSnapshots. (#1441)
add 55c9df8 HDDS-4290. Enable insight point for SCM heartbeat protocol (#1453)
add 60d2bcc HDDS-4274. Change the log level of the SCM Delete block to improve performance. (#1446)
add f8a62d6 HDDS-3810. Add the logic to distribute open containers among the pipelines of a datanode. (#1274)
add 5719615 HDDS-4304. Close Container event can fail if pipeline is removed first. (#1471)
add 8cd86a6 HDDS-4299. Display Ratis version with ozone version (#1464)
add cfff097 HDDS-4271. Avoid logging chunk content in Ozone Insight (#1466)
add 4ad0318 HDDS-4264. Uniform naming conventions of Ozone Shell Options. (#1447)
add d6d27e4 HDDS-4242. Copy PrefixInfo proto to new project hadoop-ozone/interface-storage (#1444)
add 19cb481 HDDS-4156. add hierarchical layout to Chinese doc (#1368)
add 5ab169a Merge master into decom branch
No new revisions were added by this update.
Summary of changes:
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 8 +-
.../RequestTypeDependentRetryPolicyCreator.java | 8 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../{VersionInfo.java => RatisVersionInfo.java} | 66 +--
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 9 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 34 +-
.../container/common/helpers/ChunkInfoList.java | 56 ++
.../common/src/main/resources/ozone-default.xml | 7 +
.../container/common/helpers/ContainerUtils.java | 66 ++-
.../container/common/impl/ContainerDataYaml.java | 3 +
.../container/common/impl/HddsDispatcher.java | 4 +-
.../container/common/interfaces/BlockIterator.java | 5 +-
.../container/common/interfaces/Container.java | 7 -
.../CloseContainerCommandHandler.java | 2 +-
.../CreatePipelineCommandHandler.java | 2 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 69 ++-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../transport/server/ratis/XceiverServerRatis.java | 4 +-
.../container/common/utils/ContainerCache.java | 34 +-
.../container/common/utils/ReferenceCountedDB.java | 10 +-
.../container/keyvalue/KeyValueBlockIterator.java | 156 -----
.../container/keyvalue/KeyValueContainer.java | 26 +-
.../container/keyvalue/KeyValueContainerCheck.java | 12 +-
.../container/keyvalue/KeyValueContainerData.java | 48 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 194 ++++---
.../container/keyvalue/impl/BlockManagerImpl.java | 65 ++-
.../background/BlockDeletingService.java | 53 +-
.../metadata/AbstractDatanodeDBDefinition.java | 74 +++
.../container/metadata/AbstractDatanodeStore.java | 297 ++++++++++
.../ozone/container/metadata/BlockDataCodec.java} | 40 +-
.../container/metadata/ChunkInfoListCodec.java} | 38 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 91 +++
.../metadata/DatanodeSchemaTwoDBDefinition.java | 81 +++
.../ozone/container/metadata/DatanodeStore.java | 94 ++++
.../metadata/DatanodeStoreSchemaOneImpl.java | 49 ++
.../metadata/DatanodeStoreSchemaTwoImpl.java | 47 +-
.../ozone/container/metadata/DatanodeTable.java | 130 +++++
.../metadata/SchemaOneChunkInfoListCodec.java | 68 +++
.../metadata/SchemaOneDeletedBlocksTable.java | 180 ++++++
.../container/metadata/SchemaOneKeyCodec.java | 106 ++++
.../ozone/container/metadata}/package-info.java | 9 +-
.../container/common/TestBlockDeletingService.java | 110 +++-
.../ozone/container/common/TestContainerCache.java | 25 +-
.../common/TestKeyValueContainerData.java | 3 +
.../TestSchemaOneBackwardsCompatibility.java | 626 +++++++++++++++++++++
.../common/impl/TestContainerDataYaml.java | 3 +
.../keyvalue/TestKeyValueBlockIterator.java | 340 +++++++----
.../container/keyvalue/TestKeyValueContainer.java | 75 +--
.../keyvalue/TestKeyValueContainerCheck.java | 18 +-
.../container/ozoneimpl/TestContainerReader.java | 43 +-
.../container/ozoneimpl/TestOzoneContainer.java | 14 +-
.../test/resources/123-dn-container.db/000024.sst | Bin 0 -> 1022 bytes
.../test/resources/123-dn-container.db/000026.sst | Bin 0 -> 827 bytes
.../test/resources/123-dn-container.db/000032.sst | Bin 0 -> 896 bytes
.../test/resources/123-dn-container.db/000034.log | 0
.../src/test/resources/123-dn-container.db/CURRENT | 1 +
.../test/resources/123-dn-container.db/IDENTITY | 1 +
.../resources/123-dn-container.db/MANIFEST-000033 | Bin 0 -> 297 bytes
.../resources/123-dn-container.db/OPTIONS-000033 | 165 ++++++
.../resources/123-dn-container.db/OPTIONS-000036 | 165 ++++++
.../src/test/resources/123.container | 10 +
hadoop-hdds/docs/content/concept/Datanodes.zh.md | 3 +
hadoop-hdds/docs/content/concept/Overview.zh.md | 5 +
.../docs/content/concept/OzoneManager.zh.md | 3 +
.../content/concept/StorageContainerManager.zh.md | 3 +
hadoop-hdds/docs/content/concept/_index.zh.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 3 +
hadoop-hdds/docs/content/interface/JavaApi.zh.md | 3 +
hadoop-hdds/docs/content/interface/O3fs.zh.md | 3 +
hadoop-hdds/docs/content/interface/S3.zh.md | 3 +
.../docs/content/security/SecureOzone.zh.md | 3 +
hadoop-hdds/docs/content/security/SecuringS3.zh.md | 3 +
.../docs/content/security/SecuringTDE.zh.md | 3 +
.../docs/content/security/SecurityAcls.zh.md | 3 +
.../docs/content/security/SecurityWithRanger.zh.md | 3 +
hadoop-hdds/docs/content/tools/TestTools.md | 14 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 14 +-
.../themes/ozonedoc/layouts/partials/navbar.html | 4 +-
.../docs/themes/ozonedoc/static/css/ozonedoc.css | 11 +
.../server/OzoneProtocolMessageDispatcher.java | 32 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 42 +-
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 14 +-
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 8 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 41 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 17 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 92 +++
.../org/apache/hadoop/hdds/utils/db/Table.java | 56 ++
.../apache/hadoop/hdds/utils/db/TypedTable.java | 46 ++
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
hadoop-hdds/pom.xml | 2 +
.../container/AbstractContainerReportHandler.java | 51 +-
.../hdds/scm/container/ContainerReportHandler.java | 12 +-
.../IncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 100 +++-
.../hdds/scm/container/SCMContainerManager.java | 29 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 33 ++
.../apache/hadoop/hdds/scm/node/NodeManager.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 53 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 4 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 7 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 6 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 16 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 6 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 1 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 69 +++
.../hadoop/hdds/scm/container/MockNodeManager.java | 28 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../scm/container/TestContainerReportHandler.java | 33 ++
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 4 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 4 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 74 +++
.../testutils/ReplicationNodeManagerMock.java | 12 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 10 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 43 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 74 ++-
.../apache/hadoop/ozone/client/OzoneVolume.java | 26 +
.../org/apache/hadoop/ozone/client/VolumeArgs.java | 10 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 4 +-
.../ozone/client/protocol/ClientProtocol.java | 15 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 72 ++-
.../ozone/om/exceptions/OMNotLeaderException.java | 2 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 54 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 54 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 15 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 4 +-
.../main/k8s/definitions/ozone/freon/freon.yaml | 2 +-
.../getting-started/freon/freon-deployment.yaml | 2 +-
.../examples/minikube/freon/freon-deployment.yaml | 2 +-
.../examples/ozone-dev/freon/freon-deployment.yaml | 2 +-
.../k8s/examples/ozone/freon/freon-deployment.yaml | 2 +-
.../main/smoketest/auditparser/auditparser.robot | 2 +-
.../dist/src/main/smoketest/basic/basic.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 51 +-
.../main/smoketest/basic/ozone-shell-single.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +-
.../dist/src/main/smoketest/createbucketenv.robot | 2 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../src/main/smoketest/debug/ozone-debug.robot | 2 +-
.../dist/src/main/smoketest/freon/freon.robot | 2 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 8 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 2 +-
.../dist/src/main/smoketest/spnego/web.robot | 2 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 40 +-
.../src/test/blockade/ozone/client.py | 10 +-
.../ozone/insight/BaseInsightSubCommand.java | 3 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 4 +-
.../ozone/TestStorageContainerManagerHelper.java | 31 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 4 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 8 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../rpc/TestOzoneClientRetriesOnException.java | 2 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 128 ++++-
.../client/rpc/TestValidateBCSIDOnRestart.java | 8 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 92 ++-
.../hadoop/ozone/fsck/TestContainerMapper.java | 3 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 -
.../src/main/proto/OmClientProtocol.proto | 4 +
hadoop-ozone/interface-storage/pom.xml | 30 +
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 5 +-
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 13 +-
.../hadoop/ozone/om/helpers/OzoneAclStorage.java | 63 +++
.../ozone/om/helpers/OzoneAclStorageUtil.java | 62 ++
.../hadoop/ozone/om/helpers}/package-info.java | 4 +-
.../src/main/proto/OmStorageProtocol.proto | 60 ++
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 0
.../hadoop/ozone/om/helpers}/package-info.java | 4 +-
.../apache/hadoop/ozone/om/KeyDeletingService.java | 6 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +-
.../om/request/bucket/OMBucketCreateRequest.java | 38 ++
.../request/bucket/OMBucketSetPropertyRequest.java | 64 +++
.../om/request/volume/OMVolumeSetQuotaRequest.java | 54 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 13 +-
.../ozone/om/request/TestOMRequestUtils.java | 49 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 41 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 33 ++
hadoop-ozone/ozonefs-shaded/pom.xml | 5 +
.../ReconIncrementalContainerReportHandler.java | 2 +-
.../hadoop/ozone/client/ObjectStoreStub.java | 4 +-
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 15 +-
.../ozone/freon/HadoopNestedDirGenerator.java | 5 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 40 +-
...ketHandler.java => ClearSpaceQuotaOptions.java} | 26 +-
...reTypeOption.java => SetSpaceQuotaOptions.java} | 32 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 4 +-
...teBucketHandler.java => ClearQuotaHandler.java} | 30 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 13 +
...ListBucketHandler.java => SetQuotaHandler.java} | 54 +-
.../ClearQuotaHandler.java} | 30 +-
.../ozone/shell/volume/CreateVolumeHandler.java | 21 +-
.../SetQuotaHandler.java} | 49 +-
.../ozone/shell/volume/UpdateVolumeHandler.java | 23 -
.../hadoop/ozone/shell/volume/VolumeCommands.java | 4 +-
pom.xml | 2 +-
202 files changed, 5347 insertions(+), 1301 deletions(-)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/{VersionInfo.java => RatisVersionInfo.java} (50%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeDBDefinition.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java => container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/BlockDataCodec.java} (52%)
copy hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java => container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/ChunkInfoListCodec.java} (53%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStore.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaOneImpl.java
copy hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneKMSUtil.java => hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaTwoImpl.java (52%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeTable.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/SchemaOneChunkInfoListCodec.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/SchemaOneDeletedBlocksTable.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/SchemaOneKeyCodec.java
copy hadoop-hdds/{client/src/main/java/org/apache/hadoop/hdds/scm/client => container-service/src/main/java/org/apache/hadoop/ozone/container/metadata}/package-info.java (81%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/000024.sst
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/000026.sst
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/000032.sst
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/000034.log
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/CURRENT
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/IDENTITY
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/MANIFEST-000033
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/OPTIONS-000033
create mode 100644 hadoop-hdds/container-service/src/test/resources/123-dn-container.db/OPTIONS-000036
create mode 100644 hadoop-hdds/container-service/src/test/resources/123.container
rename hadoop-ozone/{common => interface-storage}/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java (92%)
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclStorage.java
create mode 100644 hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclStorageUtil.java
copy hadoop-ozone/interface-storage/src/{test/java/org/apache/hadoop/ozone/om/codec => main/java/org/apache/hadoop/ozone/om/helpers}/package-info.java (91%)
create mode 100644 hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto
rename hadoop-ozone/{common => interface-storage}/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java (100%)
copy hadoop-ozone/interface-storage/src/{main/java/org/apache/hadoop/ozone/om => test/java/org/apache/hadoop/ozone/om/helpers}/package-info.java (92%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/{bucket/BucketHandler.java => ClearSpaceQuotaOptions.java} (63%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/{StoreTypeOption.java => SetSpaceQuotaOptions.java} (59%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/{DeleteBucketHandler.java => ClearQuotaHandler.java} (66%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/{ListBucketHandler.java => SetQuotaHandler.java} (54%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/{bucket/DeleteBucketHandler.java => volume/ClearQuotaHandler.java} (66%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/{bucket/ListBucketHandler.java => volume/SetQuotaHandler.java} (57%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org