You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/11/30 19:49:03 UTC
[ozone] branch HDDS-3698-upgrade updated (bf7624b -> 6b9ed19)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from bf7624b Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add 51ffc82 HDDS-4497. Recon File Size Count task throws SQL Exception. (#1612)
add 1b2f2ef HDDS-3689. Add various profiles to MiniOzoneChaosCluster to run different modes. (#1420)
add a9ff68a HDDS-4492. CLI flag --quota should default to 'spaceQuota' to preserve backward compatibility. (#1609)
add 4b69f08 HDDS-4501. Reload OM State fail should terminate OM for any exceptions. (#1622)
add 1a304ba HDDS-4392. [DOC] Add Recon architecture to docs (#1602)
add 54cca0b HDDS-4308. Fix issue with quota update (#1489)
add a4cd12c HDDS-4471. GrpcOutputStream length can overflow (#1617)
add fdb373f HDDS-4487. SCM can avoid using RETRIABLE_DATANODE_COMMAND for datanode deletion commands. (#1621)
add d83ec1a HDDS-4481. With HA OM can send deletion blocks to SCM multiple times. (#1608)
add 1235430 HDDS-4512. Remove unused netty3 transitive dependency (#1627)
add 43fdd71 HDDS-4370. Datanode deletion service can avoid storing deleted blocks. (#1620)
add 5704341 HDDS-3363. Intermittent failure in testContainerImportExport (#1618)
add 143f076 HDDS-4510. SCM can avoid creating RetriableDatanodeEventWatcher for deletion command ACK (#1626)
add 130ba4d HDDS-4511: Avoiding StaleNodeHandler to take effect in TestDeleteWithSlowFollower. (#1625)
add 536a6e7 HDDS-4461. Reuse compiled binaries in acceptance test (#1588)
add 9c6f805 HDDS-4429. Create unit test for SimpleContainerDownloader. (#1551)
add 6b9ed19 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
No new revisions were added by this update.
Summary of changes:
.github/workflows/post-commit.yml | 90 ++--
hadoop-hdds/client/pom.xml | 5 -
hadoop-hdds/container-service/pom.xml | 11 +-
.../ozone/container/common/volume/HddsVolume.java | 4 +-
.../container/keyvalue/KeyValueContainer.java | 3 +
.../background/BlockDeletingService.java | 19 +-
.../container/replication/GrpcOutputStream.java | 2 +-
.../replication/SimpleContainerDownloader.java | 80 ++--
.../container/common/TestBlockDeletingService.java | 326 +++++++------
.../TestSchemaOneBackwardsCompatibility.java | 83 ++--
.../container/keyvalue/TestKeyValueContainer.java | 1 -
.../replication/TestSimpleContainerDownloader.java | 127 ++++-
hadoop-hdds/docs/content/concept/OzoneManager.md | 4 +-
hadoop-hdds/docs/content/concept/Recon.md | 163 +++++++
.../docs/content/concept/ReconHighLevelDesign.png | Bin 0 -> 239168 bytes
hadoop-hdds/docs/content/concept/ReconOmDesign.png | Bin 0 -> 162797 bytes
.../docs/content/concept/ReconScmDesign.png | Bin 0 -> 181628 bytes
.../content/concept/StorageContainerManager.md | 5 +-
hadoop-hdds/docs/content/feature/Quota.md | 9 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 8 +-
hadoop-hdds/docs/content/feature/Recon.md | 26 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 511 +++++++++++++++++++++
hadoop-hdds/framework/pom.xml | 20 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 6 +
hadoop-hdds/hadoop-dependency-test/pom.xml | 12 +
hadoop-hdds/pom.xml | 3 +
hadoop-hdds/server-scm/pom.xml | 6 +
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 9 -
.../apache/hadoop/ozone/client/OzoneVolume.java | 17 -
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 22 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 33 +-
hadoop-ozone/datanode/pom.xml | 2 +-
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 5 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 8 +-
.../apache/hadoop/ozone/OzoneChaosCluster.java} | 37 +-
.../ozone/TestAllMiniChaosOzoneCluster.java} | 32 +-
.../ozone/TestDatanodeMiniChaosOzoneCluster.java} | 34 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 100 ++--
.../TestOzoneManagerMiniChaosOzoneCluster.java} | 38 +-
.../hadoop/ozone/failure/FailureManager.java | 3 +-
.../org/apache/hadoop/ozone/failure/Failures.java | 13 +
.../ozone/loadgenerators/AgedDirLoadGenerator.java | 1 -
.../ozone/loadgenerators/AgedLoadGenerator.java | 6 +-
.../loadgenerators/FilesystemLoadGenerator.java | 1 -
.../{utils => loadgenerators}/LoadBucket.java | 2 +-
.../hadoop/ozone/loadgenerators/LoadGenerator.java | 17 +
.../loadgenerators/NestedDirLoadGenerator.java | 1 -
.../loadgenerators/RandomDirLoadGenerator.java | 1 -
.../ozone/loadgenerators/RandomLoadGenerator.java | 1 -
.../loadgenerators/ReadOnlyLoadGenerator.java | 1 -
.../apache/hadoop/ozone/utils/TestProbability.java | 43 --
.../src/test/resources/log4j.properties | 2 +-
hadoop-ozone/integration-test/pom.xml | 6 +
.../client/rpc/TestDeleteWithSlowFollower.java | 7 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 296 +-----------
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 47 ++
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 77 ++++
.../ozone/om/TestOzoneManagerHAWithData.java | 37 --
.../hadoop/ozone/shell/TestOzoneShellHA.java | 42 ++
.../src/main/proto/OmClientProtocol.proto | 1 -
hadoop-ozone/interface-storage/pom.xml | 14 +-
hadoop-ozone/ozone-manager/pom.xml | 6 +
.../apache/hadoop/ozone/om/KeyDeletingService.java | 19 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 4 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 34 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 15 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 8 +-
.../om/request/key/OMAllocateBlockRequest.java | 16 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 7 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 7 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 23 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 7 +-
.../multipart/S3MultipartUploadAbortRequest.java | 5 +-
.../S3MultipartUploadCommitPartRequest.java | 10 +-
.../om/response/key/OMAllocateBlockResponse.java | 4 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 4 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 4 -
.../ozone/om/response/key/OMKeyDeleteResponse.java | 4 -
.../om/response/key/OMKeysDeleteResponse.java | 4 -
.../multipart/S3MultipartUploadAbortResponse.java | 4 -
.../S3MultipartUploadCommitPartResponse.java | 5 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 3 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 3 +-
hadoop-ozone/pom.xml | 7 +
hadoop-ozone/recon/pom.xml | 6 +
.../persistence/DefaultDataSourceProvider.java | 6 +
.../recon/persistence/DerbyDataSourceProvider.java | 1 -
.../ozone/recon/tasks/FileSizeCountTask.java | 4 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 4 +-
.../persistence/TestReconWithDifferentSqlDBs.java | 5 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 6 +
hadoop-ozone/tools/pom.xml | 2 +-
.../hadoop/ozone/shell/SetSpaceQuotaOptions.java | 3 +-
pom.xml | 6 -
99 files changed, 1721 insertions(+), 1033 deletions(-)
create mode 100644 hadoop-hdds/docs/content/concept/Recon.md
create mode 100644 hadoop-hdds/docs/content/concept/ReconHighLevelDesign.png
create mode 100644 hadoop-hdds/docs/content/concept/ReconOmDesign.png
create mode 100644 hadoop-hdds/docs/content/concept/ReconScmDesign.png
create mode 100644 hadoop-hdds/docs/content/interface/ReconApi.md
copy hadoop-ozone/{tools/src/main/java/org/apache/hadoop/ozone/segmentparser/RatisLogParser.java => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/OzoneChaosCluster.java} (60%)
copy hadoop-ozone/{tools/src/main/java/org/apache/hadoop/ozone/segmentparser/GenericRatisLogParser.java => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java} (64%)
copy hadoop-ozone/{tools/src/main/java/org/apache/hadoop/ozone/segmentparser/GenericRatisLogParser.java => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java} (61%)
copy hadoop-ozone/{tools/src/main/java/org/apache/hadoop/ozone/segmentparser/GenericRatisLogParser.java => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java} (54%)
rename hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/{utils => loadgenerators}/LoadBucket.java (99%)
delete mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/TestProbability.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org