You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2020/06/03 23:15:32 UTC
[hadoop-ozone] branch HDDS-2665-ofs updated (d6d31c6 -> b9dac41)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from d6d31c6 HDDS-2969. Implement ofs://: Add contract test (#865)
add 0e3f807 HDDS-3480. TestSCMNodeMetrics is flaky (#860)
add a53209e HDDS-3352. Support for native ozone filesystem client using libhdfs. (#782)
add c973bac HDDS-3461. Add acceptance test to smoketest CSI service startup
add ab0e66e HDDS-3373. Intermittent failure in TestOMRatisLogParser (#867)
add 76e3460 HDDS-3222. Add integration test for Recon FSCK. (#850)
add cb35ed8 Revert "HDDS-3461. Add acceptance test to smoketest CSI service startup"
add 4b1fa10 HDDS-3465. OM Failover retry happens too quickly when new leader suggested and retrying on same OM. (#859)
add 81a2886 HDDS-3318. Addendum: Support /close command in the Github comments. (HDDS-3489)
add f7fb073 HDDS-2569. Handle InterruptedException in LogStreamServlet
add e4febef HDDS-3455. Change MiniLoadGenerator to a pluggable model. (#842)
add 68fc181 HDDS-3488. Remove unnecessary jackson dependencies from Ozone
add 2791d71 HDDS-3487. Ozone start fails with NullPointerException in TLS enabled cluster (#871)
add ea94276 HDDS-3490. Remove avro as a dependency in Ozone. (#873)
add eeb7ea2 HDDS-3375. S3A failing complete multipart upload with Ozone S3. (#807)
add ade09f1 HDDS-3315. Use EventQueue for delayed/immediate safe mode rule notification
add 9042944 HDDS-3504. Disable flaky topology related acceptance test until the fix.
add 51e9cfa HDDS-3506. Disable TestOzoneFileInterfaces.testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile during the fix
add 1293258 HDDS-3321. Prometheus endpoint should have an option to be configured with Token based authentication.
add b4f594d HDDS-3398. Update Ratis/gRPC/Netty dependencies. (#879)
add 5fbb045 HDDS-3223. Improve s3g read 1GB object efficiency by 100 times (#843)
add 693a88c HDDS-3507. Disable Jaeger tracing by default. (#883)
add 099e808 HDDS-3443. Enable TestOmMetrics#testBucketOps (#882)
add 1a51d58 HDDS-3314. scmcli container info command failing intermittently (#833)
add 8ce8efa HDDS-2406. Ozone shell key get throws IllegalArgumentException if pipeline is empty. (#884)
add a7a754b HDDS-3469. Add third party jar versions as properties in pom.xml (#878)
add d849904 HDDS-3482. Impl getUriDefaultPort to BasicOzFsOzFs, return -1 forever to pass the checkPath
add f2b5b10 HDDS-3053. Decrease the number of the chunk writer threads
add b5d6743 HDDS-3385. Simplify S3 -> Ozone volume mapping
add 3a1681b HDDS-3411. Switch Recon SQL DB to Derby. (#839)
add 72bb9b7 HDDS-3515. Ensure consistent OM token service field in HA environment. (#889)
add 3271ad0 HDDS-3495. Clarify the document of OzoneFs (#875)
add 7f66b06 HDDS-3502. Remove dependence on commons-lang (#888)
add b0051d7 HDDS-3418. Enable TestNodeFailure test cases. (#848)
add f20cb4e HDDS-2424. Add the recover-trash command server side handling. (#399)
add 69ee3a3 HDDS-3178. Add unit tests for OMGetDelegationToken Request and Response (#770)
add e4ee2f3 HDDS-3449. Enable TestSCMPipelineMetrics test cases (#892)
add b2f2d2b HDDS-2796. beyond/_index.md translation (#523)
add 22f1712 HDDS-3504. Topology Test is Intermitent. Reduce container size to 256MB and blocksize to 64MB (#891)
add 8576072 HDDS-3457. When ACL enable, use ozonefs put key will get OMException: Key not found, checkAccess failed.
add 4a9fde5 HDDS-3282. ozone.http.filter.initializers can't be set properly for S… (#724)
add 946cca5 HDDS-3226. Tracing Ozone Manager DB write batch operations. (#691)
add 92cf63c HDDS-3513. Add OzoneConfiguration to UGI when startup S3Gateway (#897)
add 7c6ba28 HDDS-3333. Recon UI: All the pages should auto reload (#896)
add a2ab8d6 HDDS-3217. Datanode startup is slow due to iterating container DB 2-3 times. (#742)
add ad229ea HDDS-3553. remove hard-code configuration in Gateway.java (#901)
add 6424b46 HDDS-3492. Cp command not working on Ozone mounted using hdfs-fuse. (#887)
add 6cf238d HDDS-3558. Recon UI: Add strict linter rules to improve code quality (#905)
add ae8d15c HDDS-3517. Add a directory based Ozone Manager LoadGenerator. (#893)
add fd3dc8a HDDS-2576. Handle InterruptedException in ratis related files
add 6fb2b3e HDDS-3501. OzoneFileStatus should not extend FileStatus (#894)
add 82f97c2 HDDS-3560 OMFailoverProxyProvider throws IllegalAccessError when assigning proxy with Hadoop version < 3.2. (#904)
add 9f6d72f HDDS-3573. Update Ozone dependency hadoop.version to hadoop-3.2.1. (#916)
add eb7554a HDDS-3417. Enable TestSCMNodeManager#testScmNodeReportUpdate (#910)
add 53e7018 HDDS-3543. Remove unused joda-time (#914)
add be8a2fc HDDS-3549. TestKeyInputStream#testSeek fails intermittently. (#899)
add f81ce3a HDDS-3081. Replication manager should detect and correct containers which don't meet the placement policy (#881)
add 15910a1 HDDS-3327. Fix s3api create bucket BUCKET_NOT_FOUND and acl initialization problem when enable acl. (#880)
add e4964d9 Handle InterruptedException sonar issues in ContainerMetadataScanner (#908)
add 8849561 HDDS-3453. Use UrlConnectionFactory to handle HTTP Client SPNEGO for … (#903)
add 6ee5f4b HDDS-3389. Add response to SetVolumePropertyResponse proto (#876)
add 34debd3 HDDS-3571. Recon: Display leader count in Datanodes page (#919)
add a2d650a HDDS-3473. Ozone chunkinfo CLI should display block file path info (#886)
add f046ebf HDDS-3602. Fix KeyInputStream by adding a timeout exception. (#931)
add eac144d HDDS-3461. Add acceptance test to smoketest CSI service startup
add 9eb3f7d HDDS-3391. Delegate admin ACL checks to Ozone authorizer plugin. (#923)
add 3ec451a HDDS-3613. Fix JVMPause monitor start in OzoneManager. (#939)
add a75c2f6 HDDS-3588. Fix NPE while getPipelines if absent in query2OpenPipelines (#922)
add 676610e HDDS-3493. Refactor Failures in MiniOzoneChaosCluster into pluggable model. (#874)
add 9023fe7 HDDS-3606. Add datanode port into the printTopology command output (#935)
add 6045316 HDDS-3563. Make /s3v volume configurable instead of constant.
add cf6631e HDDS-3597. using protobuf maven plugin instead of the legacy protoc executable file
add 42a5630 HDDS-3595. Add a maven proto file backward compatibility checker into Ozone.
add dbf518e HDDS-3625. BadRequestException interception doesn't take effect in s3g. (#948)
add f55bb65 HDDS-3601. Refactor TestOzoneManagerHA.java into multiple tests to avoid frequent timeout issues
add 2e716a0 HDDS-3624. Improve error message when GC parameters are not set. (#947)
add fb8c130 HDDS-3557. Provide generic introduction / deep-dive slides as part of the documentation
add f700bcc HDDS-3634. Fix a typo in Datanodes.md (#952)
add 4ad682d HDDS-3623. Implement getReadCopy in Table. (#946)
add c221f9c HDDS-3629. Fix TestBlockOutputStreamWithFailures#test2DatanodesFailure. (#953)
add 89484f5 HDDS-3064. Get Key is hung when READ delay is injected in chunk file path.
add daebde2 HDDS-3474. Create transactionInfo Table in OmMetadataManager. (#855)
add 2ca872c HDDS-2556. Handle InterruptedException in BlockOutputStream (#942)
add 28fd39a HDDS-3638. Add a cat command to show the text of a file in the Ozone server (#954)
add 4a9b7e0 HDDS-3614. Remove S3Table from OmMetadataManager. (#940)
add 6551898 HDDS-3647. NPE while open datanode page since a pipeline no leader (#964)
add a7860d4 HDDS-3649. Add documentation for Cat Key Command (#967)
add 0781377 HDDS-2572. Handle InterruptedException in SCMSecurityProtocolServer (#960)
add 31c5027 HDDS-3635. Archive jacoco coverage files for unit/integration tests (#951)
add 9dc8ee7 HDDS-3665. Container info command should print uuid of datanode (#972)
add d1f8b3c HDDS-3663. Display uuid in recon datanode page to distinguish each datanode (#973)
add 734862e HDDS-3500. Hide OMFailoverProxyProvider usage behind an interface
add fb0da02 HDDS-3609. Avoid to use Hadoop3.x IOUtils in Ozone Client (#944)
add 2ce7407 HDDS-3646. Add a copy command to copy key to a new one. (#963)
add ece8af7 HDDS-2563. Handle InterruptedException in RunningDatanodeState (#974)
add 796a598 HDDS-3618. Update OM HA acceptance test to Python3 (#965)
add 302f4f3 HDDS-2574. Handle InterruptedException in OzoneDelegationTokenSecretManager (#961)
add d5f521b HDDS-3607. Lot of warnings at DN startup. (#937)
add b03d7b4 HDDS-2566. Handle InterruptedException in ContainerDataScanner (#971)
add 5b06294 HDDS-3657. Unregister the metrics source when replicationmanager stop. (#966)
add ec764af HDDS-3633. Improve UniqueId.next() efficiency. (#968)
add c7bda1a HDDS-3416. Enable TestSCMNodeManager#testScmStatsFromNodeReport (#909)
add b98850d HDDS-3673. Version information not printed correctly in SCM WebUI. (#977)
add b66869d HDDS-3309. Add TimedOutTestsListener to surefire and add timeout to integration tests (#813)
add d3551e2 HDDS-3475. Use transactionInfo table to persist transaction information. (#856)
add 383b078 HDDS-3550. Fix TestReadRetries. (#979)
add d4befa5 HDDS-3670. Remove property ozone.scm.container.creation.lease.timeout (#985)
add 9d89446 HDDS-3645. Add a replication type option for putkey command (#962)
add 5844712 HDDS-2764. security/_index.md translation (#866)
add 4b2dc09 HDDS-3608. NPE while process a pipeline report when PipelineQuery absent in query2OpenPipelines (#938)
add ad04f31 HDDS-3691. Update gRPC to 1.29.0. (#998)
add b9e36b6 HDDS-3690. Fix a typo in BlockManagerImpl#allocateBlock (#996)
add 764af7c HDDS-3674. add a ozone-style.xml that can be imported into IDEA. (#978)
add 1754ed1 HDDS-3692: Consider avoiding stream/map/sum in write hotpath (#1003)
add 3ab76ce HDDS-2371. Print Ozone version during startup instead of Hadoop version (#869)
add e1ef54d HDDS-3672. Ozone fs failed to list intermediate directory. (#976)
add 13d5c01 HDDS-2811. recipe/SparkOzoneFSK8s.md translation (#797)
add 4c68299 HDDS-2562. Handle InterruptedException in DatanodeStateMachine (#969)
add a0c7370 HDDS-3676. Display datanode uuid into the printTopology command output (#981)
add a06d3e1 Revert "HDDS-2562. Handle InterruptedException in DatanodeStateMachine (#969)" (#1011)
add e4f23ee HDDS-3477. Disable partial chunk write during flush() call in ozone client by default. (#957)
add b05005e HDDS-3542.Ozone chunkinfo CLI cannot connect to OM when run from non-om node. (#993)
add 9007b47 HDDS-3678. Remove usage of DFSUtil.addPBProtocol method (#987)
add 2e14e40 HDDS-3518: Add a freon generator to create directory tree with files (#895)
add 1b09b63 HDDS-3694. Reduce dn-audit log (#1001)
add a00dc18 Merge remote-tracking branch 'asf/master' into HDDS-2665-ofs
new b9dac41 HDDS-3709. Rebase OFS branch - 3. Adapt to HDDS-3501 (#1015)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/comment-commands/close.sh | 2 +-
CONTRIBUTING.md | 20 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 11 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 11 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 10 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 82 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 7 +-
hadoop-hdds/common/pom.xml | 107 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 10 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 11 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 77 +-
.../hadoop/hdds/conf/HddsPrometheusConfig.java | 44 +
.../hadoop/hdds/conf/OzoneConfiguration.java | 1 -
.../hadoop/hdds/scm/ContainerPlacementStatus.java | 57 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 10 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 30 +-
.../utils/LegacyHadoopConfigurationSource.java | 6 +
.../org/apache/hadoop/hdds/utils/SignalLogger.java | 93 +
.../org/apache/hadoop/hdds/utils/UniqueId.java | 10 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 20 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 26 +
hadoop-hdds/common/src/main/proto/proto.lock | 3640 ++++++++++++++
.../common/src/main/resources/ozone-default.xml | 229 +-
.../hadoop/hdds/conf/ConfigurationSource.java | 6 +-
hadoop-hdds/container-service/pom.xml | 50 +-
.../hadoop/ozone/HddsDatanodeHttpServer.java | 10 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 9 +-
.../container/common/impl/ChunkLayOutVersion.java | 16 +-
.../ozone/container/common/impl/ContainerData.java | 10 +
.../container/common/impl/HddsDispatcher.java | 46 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 35 +-
.../states/datanode/RunningDatanodeState.java | 8 +-
.../transport/server/ratis/XceiverServerRatis.java | 10 +-
.../container/common/utils/ReferenceCountedDB.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 24 +-
.../container/keyvalue/KeyValueContainer.java | 1 -
.../container/keyvalue/KeyValueContainerData.java | 29 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 136 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 49 +-
.../background/BlockDeletingService.java | 22 +-
.../container/ozoneimpl/ContainerDataScanner.java | 14 +-
.../ozoneimpl/ContainerMetadataScanner.java | 12 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 70 +-
.../container-service/src/main/proto/proto.lock | 1070 ++++
.../ozone/container/ContainerTestHelper.java | 4 +-
.../ozone/container/common/SCMTestUtils.java | 4 +-
.../container/common/TestBlockDeletingService.java | 41 +-
.../keyvalue/TestKeyValueBlockIterator.java | 4 +-
.../container/keyvalue/TestKeyValueContainer.java | 5 +
.../keyvalue/TestKeyValueContainerCheck.java | 6 +-
.../container/ozoneimpl/TestContainerReader.java | 222 +
.../container/ozoneimpl/TestOzoneContainer.java | 7 +
hadoop-hdds/docs/content/beyond/_index.zh.md | 27 +
hadoop-hdds/docs/content/concept/Datanodes.md | 2 +-
hadoop-hdds/docs/content/interface/OzoneFS.md | 15 +-
hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 12 +-
hadoop-hdds/docs/content/interface/S3.md | 30 +-
hadoop-hdds/docs/content/interface/S3.zh.md | 21 +-
hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 17 +-
.../{SparkOzoneFSK8S.md => SparkOzoneFSK8S.zh.md} | 87 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 4 +-
hadoop-hdds/docs/content/security/_index.zh.md | 33 +
hadoop-hdds/docs/content/shell/KeyCommands.md | 18 +
hadoop-hdds/docs/content/shell/KeyCommands.zh.md | 18 +
.../docs/content/start/StartFromDockerHub.md | 11 +-
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/docs/static/slides/README.md | 25 +
hadoop-hdds/docs/static/slides/ozone-slides.drawio | 1 +
hadoop-hdds/docs/static/slides/ozone.odp | Bin 0 -> 2034839 bytes
hadoop-hdds/docs/static/slides/ozone.pdf | Bin 0 -> 1109486 bytes
hadoop-hdds/framework/pom.xml | 6 -
.../hadoop/hdds/server/events/EventQueue.java | 21 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 96 +-
.../hadoop/hdds/server/http/HttpServer2.java | 22 +-
.../hadoop/hdds/server/http/LogStreamServlet.java | 5 +-
.../hadoop/hdds/server/http/PrometheusServlet.java | 15 +
.../hdds/server/http/StaticUserWebFilter.java | 157 -
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 15 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 5 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 17 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 39 +-
.../hdds/server/http/TestBaseHttpServer.java | 10 +
hadoop-hdds/hadoop-dependency-server/pom.xml | 20 +
hadoop-hdds/pom.xml | 28 +-
hadoop-hdds/server-scm/pom.xml | 6 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 68 +
.../apache/hadoop/hdds/scm/block/BlockManager.java | 16 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 16 +-
.../hdds/scm/container/ReplicationManager.java | 159 +-
.../ContainerPlacementStatusDefault.java | 60 +
.../algorithms/SCMContainerPlacementRackAware.java | 7 +
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 33 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 17 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 15 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 6 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 14 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 7 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 32 +-
.../hadoop/hdds/scm/safemode/SafeModeHandler.java | 141 -
.../hdds/scm/safemode/SafeModeNotification.java | 25 -
.../hdds/scm/server/SCMClientProtocolServer.java | 12 +-
.../hdds/scm/server/SCMHTTPServerConfig.java | 14 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 30 +-
.../hdds/scm/server/StorageContainerManager.java | 22 +-
.../server/StorageContainerManagerHttpServer.java | 10 +
.../scm/server/StorageContainerManagerStarter.java | 7 +-
.../scm/TestStorageContainerManagerHttpServer.java | 1 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 9 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 4 +-
.../hdds/scm/container/TestReplicationManager.java | 136 +
.../scm/container/TestSCMContainerManager.java | 6 -
.../algorithms/TestContainerPlacementFactory.java | 7 +
.../TestContainerPlacementStatusDefault.java | 70 +
.../TestSCMContainerPlacementRackAware.java | 72 +-
.../TestSCMContainerPlacementRandom.java | 40 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 34 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 75 +
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 16 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 55 +
.../hdds/scm/safemode/TestSafeModeHandler.java | 187 -
.../scm/server/TestSCMClientProtocolServer.java | 79 -
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 204 +
hadoop-hdds/test-utils/pom.xml | 1 -
hadoop-hdds/tools/pom.xml | 1 -
.../hdds/scm/cli/ContainerOperationClient.java | 2 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 29 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 7 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 173 +-
.../apache/hadoop/ozone/client/OzoneClient.java | 13 +
.../hadoop/ozone/client/OzoneClientFactory.java | 8 +-
.../ozone/client/OzoneClientInvocationHandler.java | 62 -
.../apache/hadoop/ozone/client/OzoneVolume.java | 11 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 61 +
.../ozone/client/protocol/ClientProtocol.java | 106 +-
.../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 169 +-
hadoop-ozone/common/pom.xml | 54 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 20 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 9 +-
.../hadoop/ozone/om/exceptions/OMException.java | 6 -
.../ozone/om/ha/OMFailoverProxyProvider.java | 20 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 3 +
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 231 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 68 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 269 +
.../om/protocolPB/Hadoop3OmTransportFactory.java | 24 +-
.../hadoop/ozone/om/protocolPB/OmTransport.java | 33 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 63 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 354 +-
hadoop-ozone/common/src/main/proto/FSProtos.proto | 78 -
.../src/main/proto/OzoneManagerProtocol.proto | 70 +-
hadoop-ozone/common/src/main/proto/proto.lock | 3240 ++++++++++++
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 195 +
hadoop-ozone/csi/pom.xml | 24 +-
.../apache/hadoop/ozone/csi/ControllerService.java | 11 +-
.../org/apache/hadoop/ozone/csi/CsiServer.java | 14 +-
hadoop-ozone/csi/src/main/proto/proto.lock | 1471 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 3 +
hadoop-ozone/dev-support/checks/unit.sh | 3 +
hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 +
hadoop-ozone/dev-support/intellij/ozone-style.xml | 57 +
.../intellij/runConfigurations/CsiServer.xml | 33 +
hadoop-ozone/dist/pom.xml | 7 +-
.../main/compose/{ozonescripts => ozone-csi}/.env | 0
.../src/main/compose/ozone-csi/docker-compose.yaml | 59 +
.../test.sh => ozone-csi/docker-config} | 44 +-
.../compose/{ozone-om-ha-s3 => ozone-csi}/test.sh | 9 +-
.../dist/src/main/compose/ozone-om-ha-s3/test.sh | 2 +-
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 2 +-
.../src/main/compose/ozone-om-ha/test_disabled.sh | 1 +
.../src/main/compose/ozone-topology/docker-config | 3 +-
.../src/main/compose/ozonesecure-mr/docker-config | 15 +-
.../{ozonesecure => ozonesecure-om-ha}/.env | 0
.../compose/ozonesecure-om-ha/docker-compose.yaml | 201 +
.../docker-config | 73 +-
.../{ozone-om-ha-s3 => ozonesecure-om-ha}/test.sh | 7 +-
.../src/main/compose/ozonesecure/docker-config | 60 +-
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 7 +-
.../src/main/smoketest/basic/ozone-shell.robot | 19 +-
.../dist/src/main/smoketest/commonlib.robot | 1 +
.../src/main/smoketest/csi.robot} | 35 +-
.../bucketcreate.robot => debug/ozone-debug.robot} | 31 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 8 +-
.../s3/{bucketcreate.robot => bucketdelete.robot} | 12 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 10 +-
.../main/smoketest/security/ozone-secure-fs.robot | 2 +-
.../dist/src/main/smoketest/spnego/web.robot | 65 +
.../dist/src/shell/hdds/hadoop-functions.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 312 +-
.../ozone/MiniOzoneDatanodeChaosCluster.java | 57 -
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 110 +-
.../hadoop/ozone/MiniOzoneOMChaosCluster.java | 132 -
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 64 +-
.../hadoop/ozone/failure/FailureManager.java | 99 +
.../org/apache/hadoop/ozone/failure/Failures.java | 147 +
.../apache/hadoop/ozone/failure/package-info.java} | 6 +-
...oadGenerator.java => AgedDirLoadGenerator.java} | 36 +-
.../ozone/loadgenerators/AgedLoadGenerator.java | 1 -
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 42 +-
.../hadoop/ozone/loadgenerators/LoadGenerator.java | 10 +
...dGenerator.java => NestedDirLoadGenerator.java} | 48 +-
...dGenerator.java => RandomDirLoadGenerator.java} | 30 +-
.../ozone/loadgenerators/RandomLoadGenerator.java | 15 +-
.../loadgenerators/ReadOnlyLoadGenerator.java | 10 +-
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 52 +
.../src/test/resources/log4j.properties | 3 +-
.../fault-injection-test/network-tests/pom.xml | 1 -
hadoop-ozone/insight/pom.xml | 1 -
hadoop-ozone/integration-test/pom.xml | 5 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 13 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 19 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 37 +
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 8 +
.../metrics/TestSCMContainerManagerMetrics.java | 8 +
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 8 +
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 118 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 8 +
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 8 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 12 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 12 +-
.../hadoop/ozone/TestContainerOperations.java | 8 +
.../TestContainerStateMachineIdempotency.java | 8 +
.../apache/hadoop/ozone/TestDelegationToken.java | 406 ++
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 8 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 12 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 170 +-
.../ozone/TestStorageContainerManagerHelper.java | 12 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java | 8 +
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 8 +
.../ozone/client/rpc/TestBlockOutputStream.java | 167 +-
...m.java => TestBlockOutputStreamFlushDelay.java} | 219 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 16 +-
...stBlockOutputStreamWithFailuresFlushDelay.java} | 41 +-
.../rpc/TestCloseContainerHandlingByClient.java | 8 +
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 8 +
.../client/rpc/TestContainerStateMachine.java | 10 +
.../rpc/TestContainerStateMachineFailures.java | 4 +-
...va => TestContainerStateMachineFlushDelay.java} | 111 +-
.../client/rpc/TestFailureHandlingByClient.java | 11 +-
... => TestFailureHandlingByClientFlushDelay.java} | 159 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 8 +
.../ozone/client/rpc/TestKeyInputStream.java | 78 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 8 +
.../rpc/TestOzoneClientRetriesOnException.java | 10 +
...stOzoneClientRetriesOnExceptionFlushDelay.java} | 148 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 8 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 125 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 24 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 11 +
.../ozone/client/rpc/TestWatchForCommit.java | 28 +-
.../commandhandler/TestBlockDeletion.java | 8 +-
.../commandhandler/TestCloseContainerHandler.java | 8 +
.../commandhandler/TestDeleteContainerHandler.java | 8 +
.../container/metrics/TestContainerMetrics.java | 8 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 8 +
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 8 +
.../freon/TestDataValidateWithDummyContainers.java | 8 +
.../TestDataValidateWithSafeByteOperations.java | 8 +
.../TestDataValidateWithUnsafeByteOperations.java | 8 +
.../freon/TestFreonWithDatanodeFastRestart.java | 8 +
.../ozone/freon/TestFreonWithDatanodeRestart.java | 10 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 180 +
.../hadoop/ozone/fsck/TestContainerMapper.java | 8 +
.../hadoop/ozone/om/OmFailoverProxyUtil.java | 47 +
.../ozone/om/TestContainerReportWithKeys.java | 7 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 73 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 8 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 22 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 7 +
.../hadoop/ozone/om/TestOmBlockVersioning.java | 7 +
.../org/apache/hadoop/ozone/om/TestOmInit.java | 7 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 54 +-
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 8 +
.../ozone/om/TestOzoneManagerConfiguration.java | 8 +
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 1302 +----
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 366 ++
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 330 ++
.../ozone/om/TestOzoneManagerHAWithData.java | 653 +++
.../ozone/om/TestOzoneManagerListVolumes.java | 24 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 8 +
.../ozone/om/parser/TestOMRatisLogParser.java | 16 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 14 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 15 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 132 +
.../ozone/recon/TestReconWithOzoneManager.java | 8 +
.../hadoop/ozone/scm/TestAllocateContainer.java | 7 +
.../hadoop/ozone/scm/TestContainerSmallFile.java | 7 +
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 8 +
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 8 +
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 8 +
.../hadoop/ozone/scm/TestXceiverClientManager.java | 7 +
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 8 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 177 -
.../scm/pipeline/TestPipelineManagerMXBean.java | 8 +
.../TestSCMPipelineBytesWrittenMetrics.java | 10 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 29 +-
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 8 +
.../hadoop/ozone/shell/TestOzoneShellHA.java | 28 -
.../org/apache/hadoop/ozone/shell/TestS3Shell.java | 276 --
hadoop-ozone/native-client/README.md | 96 +
.../native-client/libo3fs-examples/libo3fs_read.c | 65 +
.../native-client/libo3fs-examples/libo3fs_write.c | 91 +
hadoop-ozone/native-client/libo3fs/o3fs.c | 66 +
.../libo3fs/o3fs.h} | 41 +-
hadoop-ozone/ozone-manager/pom.xml | 1 -
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 26 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 23 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 46 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 224 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 10 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 9 +-
.../apache/hadoop/ozone/om/S3BucketManager.java | 87 -
.../hadoop/ozone/om/S3BucketManagerImpl.java | 254 -
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 2 +-
.../ozone/om/codec/OMTransactionInfoCodec.java | 50 +
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 140 +
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 179 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 15 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 41 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 9 +-
.../om/request/bucket/OMBucketCreateRequest.java | 10 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 6 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 6 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 6 +-
.../om/request/file/OMDirectoryCreateRequest.java | 9 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 8 +-
.../om/request/key/OMAllocateBlockRequest.java | 6 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 13 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 8 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 11 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 11 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 8 +-
.../om/request/key/OMTrashRecoverRequest.java | 136 +
.../om/request/key/acl/OMKeyAddAclRequest.java | 6 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 6 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 6 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 6 +-
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 6 +-
.../key/acl/prefix/OMPrefixSetAclRequest.java | 6 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 405 --
.../request/s3/bucket/S3BucketDeleteRequest.java | 217 -
.../S3InitiateMultipartUploadRequest.java | 7 +-
.../multipart/S3MultipartUploadAbortRequest.java | 7 +-
.../S3MultipartUploadCommitPartRequest.java | 7 +-
.../S3MultipartUploadCompleteRequest.java | 7 +-
.../om/request/s3/security/S3GetSecretRequest.java | 10 +-
.../security/OMCancelDelegationTokenRequest.java | 9 +-
.../security/OMGetDelegationTokenRequest.java | 11 +-
.../security/OMRenewDelegationTokenRequest.java | 9 +-
.../ozone/om/request/util/OmResponseUtil.java} | 39 +-
.../om/request/volume/OMVolumeCreateRequest.java | 25 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 6 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 11 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 5 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 5 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 5 +-
.../om/response/key/OMTrashRecoverResponse.java | 64 +
.../response/s3/bucket/S3BucketCreateResponse.java | 86 -
.../response/s3/bucket/S3BucketDeleteResponse.java | 61 -
.../ozone/om/response/s3/bucket/package-info.java | 24 -
.../om/snapshot/OzoneManagerSnapshotProvider.java | 165 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 31 +-
.../protocolPB/OzoneManagerRequestHandler.java | 45 +-
.../OzoneDelegationTokenSecretManager.java | 34 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 44 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 25 +
.../ozone/om/TestOzoneManagerHttpServer.java | 1 +
.../hadoop/ozone/om/TestS3BucketManager.java | 115 -
.../apache/hadoop/ozone/om/TestTrashService.java | 13 +-
.../ozone/om/codec/TestOMTransactionInfoCodec.java | 82 +
...tOzoneManagerDoubleBufferWithDummyResponse.java | 21 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 20 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 2 +-
.../ozone/om/request/TestOMRequestUtils.java | 34 -
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../s3/bucket/TestS3BucketCreateRequest.java | 216 -
.../s3/bucket/TestS3BucketDeleteRequest.java | 161 -
.../ozone/om/request/s3/bucket/package-info.java | 23 -
.../TestOMDelegationTokenRequest.java} | 57 +-
.../security/TestOMGetDelegationTokenRequest.java | 221 +
.../ozone/om/request/security}/package-info.java | 4 +-
.../ozone/om/response/TestOMResponseUtils.java | 43 -
.../s3/bucket/TestS3BucketCreateResponse.java | 91 -
.../s3/bucket/TestS3BucketDeleteResponse.java | 92 -
.../ozone/om/response/s3/bucket/package-info.java | 23 -
.../security/TestOMDelegationTokenResponse.java | 51 +
.../security/TestOMGetDelegationTokenResponse.java | 95 +
.../ozone/om/response/security/package-info.java} | 16 +-
.../security/acl/TestOzoneAdministrators.java | 102 +
.../security/acl/TestOzoneNativeAuthorizer.java | 95 +-
hadoop-ozone/ozonefs/pom.xml | 5 +-
.../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 4 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 42 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 48 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 1 +
.../hadoop/fs/ozone/PosixOzoneFileSystem.java | 48 +
hadoop-ozone/pom.xml | 26 +-
hadoop-ozone/recon-codegen/pom.xml | 8 +-
.../ozone/recon/codegen/JooqCodeGenerator.java | 51 +-
.../ozone/recon/codegen/ReconSqlDbConfig.java | 239 +
.../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 97 +
.../recon/schema/ContainerSchemaDefinition.java | 14 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 10 +-
.../ozone/recon/schema/StatsSchemaDefinition.java | 8 +-
.../recon/schema/UtilizationSchemaDefinition.java | 23 +-
hadoop-ozone/recon/pom.xml | 52 +-
.../hadoop/ozone/recon/ConfigurationProvider.java | 13 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 83 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 10 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 7 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 39 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 14 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 30 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 120 +-
.../ozone/recon/api/types/DatanodePipeline.java | 8 +-
.../ozone/recon/fsck/MissingContainerTask.java | 24 +-
.../recon/persistence/DataSourceConfiguration.java | 6 +-
.../persistence/DefaultDataSourceProvider.java | 11 +-
.../recon/persistence/DerbyDataSourceProvider.java | 61 +
.../recon/persistence/JooqPersistenceModule.java | 2 +-
...Provider.java => SqliteDataSourceProvider.java} | 44 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 21 +-
.../ozone/recon/scm/ReconContainerManager.java | 4 +
.../hadoop/ozone/recon/scm/ReconScmTask.java | 8 +-
.../scm/ReconStorageContainerManagerFacade.java | 14 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 58 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 69 +
.../webapps/recon/ozone-recon-web/LICENSE | 1707 +++++--
.../webapps/recon/ozone-recon-web/api/db.json | 111 +-
.../recon/ozone-recon-web/config-overrides.js | 4 +-
.../webapps/recon/ozone-recon-web/package.json | 110 +-
.../ozone-recon-web/src/{App.less => app.less} | 23 +-
.../src/{App.test.tsx => app.test.tsx} | 6 +-
.../recon/ozone-recon-web/src/{App.tsx => app.tsx} | 70 +-
.../src/components/NavBar/NavBar.tsx | 73 -
.../autoReloadPanel/autoReloadPanel.less} | 15 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 61 +
.../breadcrumbs.tsx} | 32 +-
.../{NavBar/NavBar.less => navBar/navBar.less} | 2 +-
.../src/components/navBar/navBar.tsx | 75 +
.../overviewCard.less} | 2 +-
.../overviewCard.tsx} | 75 +-
.../StorageBar.less => storageBar/storageBar.less} | 2 +-
.../StorageBar.tsx => storageBar/storageBar.tsx} | 51 +-
.../autoReload.constants.tsx} | 11 +-
.../src/constants/breadcrumbs.constants.tsx | 2 +-
.../webapps/recon/ozone-recon-web/src/index.less | 2 +-
.../webapps/recon/ozone-recon-web/src/index.tsx | 12 +-
.../ozone-recon-web/src/makeRouteWithSubRoutes.tsx | 18 +-
.../recon/ozone-recon-web/src/react-app-env.d.ts | 2 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 26 +-
.../recon/ozone-recon-web/src/serviceWorker.ts | 161 -
.../axios.types.tsx} | 8 +-
.../ozone-recon-web/src/types/datanode.types.tsx | 6 +-
.../ozone-recon-web/src/types/routes.types.tsx | 6 +-
.../StorageBar.less => utils/autoReloadHelper.tsx} | 55 +-
.../recon/ozone-recon-web/src/utils/common.tsx | 19 +-
.../recon/ozone-recon-web/src/utils/themeIcons.tsx | 69 +-
.../src/views/Datanodes/Datanodes.tsx | 198 -
.../src/views/Overview/Overview.tsx | 148 -
.../src/views/Pipelines/Pipelines.tsx | 197 -
.../Datanodes.less => datanodes/datanodes.less} | 2 +-
.../src/views/datanodes/datanodes.tsx | 252 +
.../missingContainers.less} | 2 +-
.../missingContainers.tsx} | 177 +-
.../NotFound.tsx => notFound/notFound.tsx} | 10 +-
.../Overview.less => overview/overview.less} | 4 +-
.../src/views/overview/overview.tsx | 187 +
.../Pipelines.less => pipelines/pipelines.less} | 2 +-
.../src/views/pipelines/pipelines.tsx | 215 +
.../webapps/recon/ozone-recon-web/yarn.lock | 5171 ++++++++++++++------
.../apache/hadoop/ozone/recon/TestReconUtils.java | 5 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 35 +-
.../ozone/recon/fsck/TestMissingContainerTask.java | 8 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 56 +-
.../TestReconInternalSchemaDefinition.java | 4 +-
.../persistence/TestReconWithDifferentSqlDBs.java | 159 +
.../recon/persistence/TestSqlSchemaSetup.java | 53 +
.../persistence/TestStatsSchemaDefinition.java | 4 +-
.../TestUtilizationSchemaDefinition.java | 10 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 5 +-
hadoop-ozone/s3gateway/pom.xml | 21 -
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 12 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 14 +-
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 10 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +-
.../endpoint/CompleteMultipartUploadRequest.java | 3 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 85 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 112 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 15 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 8 +-
.../s3/exception/BadRequestExceptionMapper.java} | 55 +-
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 31 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 +
.../hadoop/ozone/client/ObjectStoreStub.java | 114 +-
.../hadoop/ozone/client/OzoneClientStub.java | 5 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 10 +-
.../s3/endpoint/TestAbortMultipartUpload.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 5 +-
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 33 +-
.../hadoop/ozone/s3/endpoint/TestBucketHead.java | 12 +-
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 3 +-
...CompleteMultipartUploadRequestUnmarshaller.java | 93 +
.../s3/endpoint/TestInitiateMultipartUpload.java | 15 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 15 +-
.../TestMultiDeleteRequestUnmarshaller.java | 1 -
.../s3/endpoint/TestMultipartUploadComplete.java | 24 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 52 +-
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 8 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 18 +-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 12 +-
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 52 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 10 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 13 +-
hadoop-ozone/tools/pom.xml | 4 -
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 43 +-
.../hadoop/ozone/debug/ContainerChunkInfo.java | 10 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 6 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 1 +
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 10 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 221 +
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 8 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 4 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 4 +-
.../hadoop/ozone/genesis/BenchMarkOMClient.java | 161 -
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 9 +-
.../CatKeyHandler.java} | 45 +-
.../{PutKeyHandler.java => CopyKeyHandler.java} | 100 +-
.../hadoop/ozone/shell/keys/KeyCommands.java | 2 +
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 12 +-
.../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 4 +-
.../ozone/shell/volume/UpdateVolumeHandler.java | 6 +-
pom.xml | 376 +-
542 files changed, 28573 insertions(+), 13002 deletions(-)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsPrometheusConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ContainerPlacementStatus.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SignalLogger.java
create mode 100644 hadoop-hdds/common/src/main/proto/proto.lock
create mode 100644 hadoop-hdds/container-service/src/main/proto/proto.lock
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
create mode 100644 hadoop-hdds/docs/content/beyond/_index.zh.md
copy hadoop-hdds/docs/content/recipe/{SparkOzoneFSK8S.md => SparkOzoneFSK8S.zh.md} (51%)
create mode 100644 hadoop-hdds/docs/content/security/_index.zh.md
create mode 100644 hadoop-hdds/docs/static/slides/README.md
create mode 100644 hadoop-hdds/docs/static/slides/ozone-slides.drawio
create mode 100644 hadoop-hdds/docs/static/slides/ozone.odp
create mode 100644 hadoop-hdds/docs/static/slides/ozone.pdf
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/StaticUserWebFilter.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementStatusDefault.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeNotification.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementStatusDefault.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/Hadoop3OmTransport.java
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterInitializer.java => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/Hadoop3OmTransportFactory.java (63%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterContainer.java => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OmTransport.java (54%)
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OmTransportFactory.java
delete mode 100644 hadoop-ozone/common/src/main/proto/FSProtos.proto
create mode 100644 hadoop-ozone/common/src/main/proto/proto.lock
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
create mode 100644 hadoop-ozone/csi/src/main/proto/proto.lock
create mode 100644 hadoop-ozone/dev-support/intellij/ozone-style.xml
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/CsiServer.xml
copy hadoop-ozone/dist/src/main/compose/{ozonescripts => ozone-csi}/.env (100%)
create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml
copy hadoop-ozone/dist/src/main/compose/{ozonesecure/test.sh => ozone-csi/docker-config} (55%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/compose/{ozone-om-ha-s3 => ozone-csi}/test.sh (84%)
mode change 100644 => 100755
copy hadoop-ozone/dist/src/main/compose/{ozonesecure => ozonesecure-om-ha}/.env (100%)
create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml
copy hadoop-ozone/dist/src/main/compose/{ozonesecure => ozonesecure-om-ha}/docker-config (70%)
copy hadoop-ozone/dist/src/main/compose/{ozone-om-ha-s3 => ozonesecure-om-ha}/test.sh (91%)
mode change 100644 => 100755
copy hadoop-ozone/{dev-support/checks/integration.sh => dist/src/main/smoketest/csi.robot} (53%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/smoketest/{s3/bucketcreate.robot => debug/ozone-debug.robot} (53%)
copy hadoop-ozone/dist/src/main/smoketest/s3/{bucketcreate.robot => bucketdelete.robot} (70%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
delete mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneDatanodeChaosCluster.java
delete mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMChaosCluster.java
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java} (89%)
copy hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/{ReadOnlyLoadGenerator.java => AgedDirLoadGenerator.java} (57%)
copy hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/{ReadOnlyLoadGenerator.java => NestedDirLoadGenerator.java} (55%)
copy hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/{LoadGenerator.java => RandomDirLoadGenerator.java} (58%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{TestBlockOutputStream.java => TestBlockOutputStreamFlushDelay.java} (79%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{TestBlockOutputStreamWithFailures.java => TestBlockOutputStreamWithFailuresFlushDelay.java} (97%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{TestContainerStateMachine.java => TestContainerStateMachineFlushDelay.java} (61%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{TestMultiBlockWritesWithDnFailures.java => TestFailureHandlingByClientFlushDelay.java} (66%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{Test2BlockOutputStream.java => TestOzoneClientRetriesOnExceptionFlushDelay.java} (52%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OmFailoverProxyUtil.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithACL.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestS3Shell.java
create mode 100644 hadoop-ozone/native-client/README.md
create mode 100644 hadoop-ozone/native-client/libo3fs-examples/libo3fs_read.c
create mode 100644 hadoop-ozone/native-client/libo3fs-examples/libo3fs_write.c
create mode 100644 hadoop-ozone/native-client/libo3fs/o3fs.c
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js => native-client/libo3fs/o3fs.h} (56%)
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMTransactionInfoCodec.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketDeleteRequest.java
copy hadoop-ozone/{integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithUnsafeByteOperations.java => ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OmResponseUtil.java} (50%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketCreateResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketDeleteResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOMTransactionInfoCodec.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketDeleteRequest.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java
rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/{s3/bucket/TestS3BucketRequest.java => security/TestOMDelegationTokenRequest.java} (60%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
rename hadoop-ozone/ozone-manager/src/{main/java/org/apache/hadoop/ozone/om/request/s3/bucket => test/java/org/apache/hadoop/ozone/om/request/security}/package-info.java (87%)
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketCreateResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketDeleteResponse.java
delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
copy hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less => ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/package-info.java} (80%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneAdministrators.java
create mode 100644 hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/PosixOzoneFileSystem.java
create mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSqlDbConfig.java
create mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/{DefaultDataSourceProvider.java => SqliteDataSourceProvider.java} (51%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskConfig.java
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{App.less => app.less} (90%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{App.test.tsx => app.test.tsx} (94%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{App.tsx => app.tsx} (54%)
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{views/Datanodes/Datanodes.less => components/autoReloadPanel/autoReloadPanel.less} (84%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.tsx
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{Breadcrumbs/Breadcrumbs.tsx => breadcrumbs/breadcrumbs.tsx} (74%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{NavBar/NavBar.less => navBar/navBar.less} (99%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{OverviewCard/OverviewCard.less => overviewCard/overviewCard.less} (99%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{OverviewCard/OverviewCard.tsx => overviewCard/overviewCard.tsx} (54%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{StorageBar/StorageBar.less => storageBar/storageBar.less} (99%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/{StorageBar/StorageBar.tsx => storageBar/storageBar.tsx} (55%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{views/Datanodes/Datanodes.less => constants/autoReload.constants.tsx} (88%)
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/serviceWorker.ts
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{views/MissingContainers/MissingContainers.less => types/axios.types.tsx} (93%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{components/StorageBar/StorageBar.less => utils/autoReloadHelper.tsx} (52%)
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.tsx
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
delete mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.tsx
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{Datanodes/Datanodes.less => datanodes/datanodes.less} (99%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{MissingContainers/MissingContainers.less => missingContainers/missingContainers.less} (99%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{MissingContainers/MissingContainers.tsx => missingContainers/missingContainers.tsx} (52%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{NotFound/NotFound.tsx => notFound/notFound.tsx} (89%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{Overview/Overview.less => overview/overview.less} (97%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/{Pipelines/Pipelines.less => pipelines/pipelines.less} (99%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSqlSchemaSetup.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
rename hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.less => s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/BadRequestExceptionMapper.java} (51%)
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMClient.java
rename hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/{s3/S3BucketMapping.java => keys/CatKeyHandler.java} (50%)
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/{PutKeyHandler.java => CopyKeyHandler.java} (59%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: HDDS-3709. Rebase OFS branch - 3. Adapt to
HDDS-3501 (#1015)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit b9dac41380617e1c680da2eb789857a64274ad6b
Author: Siyao Meng <sm...@cloudera.com>
AuthorDate: Wed Jun 3 10:13:28 2020 -0700
HDDS-3709. Rebase OFS branch - 3. Adapt to HDDS-3501 (#1015)
---
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 48 ++++++++--------------
1 file changed, 18 insertions(+), 30 deletions(-)
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 171937e..3c85906 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -481,10 +480,8 @@ public class BasicRootedOzoneClientAdapterImpl
try {
OzoneBucket bucket = getBucket(ofsPath, false);
OzoneFileStatus status = bucket.getFileStatus(key);
- // Note: qualifiedPath passed in is good from
- // BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
- makeQualified(status, uri, qualifiedPath, userName);
- return toFileStatusAdapter(status);
+ return toFileStatusAdapter(status, userName, uri, qualifiedPath,
+ ofsPath.getNonKeyPath());
} catch (OMException e) {
if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
throw new FileNotFoundException(key + ": No such file or directory!");
@@ -495,15 +492,6 @@ public class BasicRootedOzoneClientAdapterImpl
}
}
- public void makeQualified(FileStatus status, URI uri, Path path,
- String username) {
- if (status instanceof OzoneFileStatus) {
- ((OzoneFileStatus) status)
- .makeQualified(uri, path,
- username, username);
- }
- }
-
@Override
public Iterator<BasicKeyInfo> listKeys(String pathStr) {
incrementCounter(Statistic.OBJECTS_LIST);
@@ -634,12 +622,8 @@ public class BasicRootedOzoneClientAdapterImpl
List<FileStatusAdapter> result = new ArrayList<>();
for (OzoneFileStatus status : statuses) {
- // Get raw path (without volume and bucket name) and remove leading '/'
- String rawPath = status.getPath().toString().substring(1);
- Path appendedPath = new Path(ofsPathPrefix, rawPath);
- Path qualifiedPath = appendedPath.makeQualified(uri, workingDir);
- makeQualified(status, uri, qualifiedPath, username);
- result.add(toFileStatusAdapter(status));
+ result.add(toFileStatusAdapter(status, username, uri, workingDir,
+ ofsPathPrefix));
}
return result;
} catch (OMException e) {
@@ -762,19 +746,23 @@ public class BasicRootedOzoneClientAdapterImpl
}
}
- private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status) {
+ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status,
+ String owner, URI defaultUri, Path workingDir, String ofsPathPrefix) {
+ OmKeyInfo keyInfo = status.getKeyInfo();
+ short replication = (short) keyInfo.getFactor().getNumber();
return new FileStatusAdapter(
- status.getLen(),
- status.getPath(),
+ keyInfo.getDataSize(),
+ new Path(ofsPathPrefix + OZONE_URI_DELIMITER + keyInfo.getKeyName())
+ .makeQualified(defaultUri, workingDir),
status.isDirectory(),
- status.getReplication(),
+ replication,
status.getBlockSize(),
- status.getModificationTime(),
- status.getAccessTime(),
- status.getPermission().toShort(),
- status.getOwner(),
- status.getGroup(),
- status.getPath(),
+ keyInfo.getModificationTime(),
+ keyInfo.getModificationTime(),
+ status.isDirectory() ? (short) 00777 : (short) 00666,
+ owner,
+ owner,
+ null,
getBlockLocations(status)
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org