You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by pi...@apache.org on 2023/01/04 23:26:28 UTC
[ozone] 01/01: Merge branch 'master' into httpfs-merge
This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch HDDS-5447-httpfs
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c7f230f0506cddbfab15988f1b08cad41d30aa53
Merge: 08b028967d 21717321c6
Author: Istvan Fajth <fa...@gmail.com>
AuthorDate: Wed Jan 4 17:02:47 2023 +0100
Merge branch 'master' into httpfs-merge
.github/workflows/ci.yml | 2 +-
.github/workflows/post-commit.yml | 1 +
.github/workflows/scheduled_ci.yml | 1 +
LICENSE.txt | 10 +-
NOTICE.txt | 2 +-
hadoop-hdds/annotations/pom.xml | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 74 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +-
.../hdds/scm/storage/AbstractDataStreamOutput.java | 131 +
.../hdds/scm/storage/BlockDataStreamOutput.java | 733 ++++++
.../hdds/scm/storage/BlockExtendedInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 5 -
.../hdds/scm/storage/ByteBufferStreamOutput.java | 57 +
.../hdds/scm/storage/MultipartInputStream.java | 161 +-
...tendedInputStream.java => PartInputStream.java} | 22 +-
.../hadoop/hdds/scm/storage/StreamBuffer.java | 59 +
.../hdds/scm/storage/StreamCommitWatcher.java | 208 ++
.../hadoop/ozone/client/io/ECBlockInputStream.java | 5 -
.../ozone/client/io/ECBlockInputStreamProxy.java | 2 +-
.../client/io/ECBlockReconstructedInputStream.java | 2 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 5 -
.../ozone/client/io/TestECBlockInputStream.java | 5 -
hadoop-hdds/common/pom.xml | 6 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 20 +
.../hadoop/hdds/client/ReplicationConfig.java | 11 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 92 +-
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 18 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +
.../scm/container/common/helpers/ExcludeList.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 7 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 14 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 35 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 26 +
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 5 +
.../helpers/ContainerCommandRequestPBHelper.java | 1 +
.../common/src/main/resources/ozone-default.xml | 92 +-
.../hadoop/ozone/audit/AuditLogTestUtils.java | 79 +
.../ozone/container/ContainerTestHelper.java | 30 +-
hadoop-hdds/config/pom.xml | 4 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
hadoop-hdds/container-service/pom.xml | 8 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 17 +-
.../ozone/container/common/impl/ContainerSet.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 21 +-
.../common/interfaces/ContainerDispatcher.java | 10 +
.../ozone/container/common/interfaces/Handler.java | 5 +
.../common/statemachine/DatanodeStateMachine.java | 11 +-
.../common/statemachine/StateContext.java | 36 +-
.../DeleteContainerCommandHandler.java | 67 +-
.../ReconstructECContainersCommandHandler.java | 6 +-
.../ReplicateContainerCommandHandler.java | 3 +-
.../states/endpoint/HeartbeatEndpointTask.java | 7 +
.../common/transport/server/XceiverServerGrpc.java | 4 +-
.../server/ratis/ContainerStateMachine.java | 79 +-
.../common/transport/server/ratis/LocalStream.java | 58 +
.../transport/server/ratis/XceiverServerRatis.java | 43 +-
.../ECReconstructionCommandInfo.java | 28 +-
.../ECReconstructionCoordinator.java | 12 +
.../ECReconstructionCoordinatorTask.java | 29 +-
.../reconstruction/ECReconstructionSupervisor.java | 11 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 52 +
.../container/keyvalue/TarContainerPacker.java | 34 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 16 +
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +
.../keyvalue/impl/KeyValueStreamDataChannel.java | 277 ++
.../keyvalue/impl/StreamDataChannelBase.java | 96 +
.../keyvalue/interfaces/ChunkManager.java | 13 +
.../replication/ContainerReplicationSource.java | 5 +-
.../replication/CopyContainerCompression.java | 78 +
.../replication/GrpcReplicationClient.java | 20 +-
.../replication/GrpcReplicationService.java | 8 +-
.../OnDemandContainerReplicationSource.java | 19 +-
.../container/replication/ReplicationServer.java | 26 +-
.../replication/ReplicationSupervisor.java | 62 +-
.../replication/ReplicationSupervisorMetrics.java | 5 +-
.../container/replication/ReplicationTask.java | 34 +-
.../replication/SimpleContainerDownloader.java | 12 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 35 +
.../container/common/TestDatanodeStateMachine.java | 8 +-
.../common/statemachine/TestStateContext.java | 85 +-
.../TestDeleteContainerCommandHandler.java | 135 +
.../states/endpoint/TestHeartbeatEndpointTask.java | 29 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 2 +-
.../TestECReconstructionSupervisor.java | 96 +-
.../container/keyvalue/TestKeyValueContainer.java | 141 +-
.../container/keyvalue/TestTarContainerPacker.java | 37 +-
.../impl/TestKeyValueStreamDataChannel.java | 313 +++
.../ReplicationSupervisorScheduling.java | 10 +-
.../replication/TestReplicationSupervisor.java | 111 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 4 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../content/feature/Streaming-Write-Pipeline.md | 136 +
hadoop-hdds/docs/content/interface/Ofs.md | 32 +-
hadoop-hdds/docs/content/interface/S3.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 2 +-
hadoop-hdds/docs/content/security/SecurityAcls.md | 9 +-
.../docs/content/security/SecurityAcls.zh.md | 7 +-
.../docs/content/security/SecurityWithRanger.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
hadoop-hdds/docs/pom.xml | 4 +-
.../themes/ozonedoc/layouts/partials/header.html | 22 +
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../framework/dev-support/findbugsExcludeFile.xml | 4 -
hadoop-hdds/framework/pom.xml | 4 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 35 +
...inerLocationProtocolClientSideTranslatorPB.java | 20 +
.../hadoop/hdds/security/ssl/KeyStoresFactory.java | 14 +-
.../hdds/security/ssl/MonitoringTimerTask.java | 6 +-
.../security/ssl/PemFileBasedKeyStoresFactory.java | 15 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 7 +-
.../security/ssl/ReloadingX509TrustManager.java | 9 +-
.../certificate/authority/DefaultCAServer.java | 108 +-
.../x509/certificate/client/CertificateClient.java | 13 +-
.../client/DefaultCertificateClient.java | 35 +
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 4 +-
.../hdds/security/x509/keys/SecurityUtil.java | 34 +
.../FixedThreadPoolWithAffinityExecutor.java | 7 +-
.../hdds/server/events/SingleThreadExecutor.java | 6 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 2 +-
.../hdds/server/http/PrometheusMetricsSink.java | 109 +-
.../hdds/server/http/RatisDropwizardExports.java | 59 +-
.../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 120 +
.../org/apache/hadoop/hdds/utils/MetricsUtil.java | 100 +
.../hdds/utils/PrometheusMetricsSinkUtil.java | 116 +
.../apache/hadoop/hdds/utils/UgiMetricsUtil.java | 68 +
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 2 +-
.../ssl/TestPemFileBasedKeyStoresFactory.java | 9 +-
.../hdds/security/x509/CertificateClientTest.java | 17 +
.../certificate/authority/TestDefaultCAServer.java | 99 +-
....java => TestPrometheusMetricsIntegration.java} | 148 +-
.../hdds/utils/TestDecayRpcSchedulerUtil.java | 108 +
.../hdds/utils/TestPrometheusMetricsSinkUtil.java | 229 ++
.../hadoop/hdds/utils/TestUgiMetricsUtil.java | 63 +
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../interface-admin/src/main/resources/proto.lock | 679 +++--
hadoop-hdds/interface-client/pom.xml | 4 +-
.../src/main/proto/DatanodeClientProtocol.proto | 14 +-
.../interface-client/src/main/resources/proto.lock | 991 ++++++--
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 +
.../interface-server/src/main/resources/proto.lock | 840 ++++--
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 115 +-
.../hdds/scm/container/ContainerManager.java | 18 +
.../hdds/scm/container/ContainerManagerImpl.java | 21 +
.../balancer/AbstractFindTargetGreedy.java | 31 +-
.../scm/container/balancer/FindSourceGreedy.java | 20 +-
.../ContainerPlacementStatusDefault.java | 36 +-
.../algorithms/SCMContainerPlacementRackAware.java | 8 +
.../SCMContainerPlacementRackScatter.java | 138 +-
.../algorithms/SCMContainerPlacementRandom.java | 3 +-
.../AbstractOverReplicationHandler.java | 4 +-
.../replication/ContainerHealthResult.java | 35 +-
.../replication/ContainerReplicaPendingOps.java | 45 +
.../ContainerReplicaPendingOpsSubscriber.java} | 27 +-
.../replication/ECMisReplicationHandler.java | 71 +
.../replication/ECUnderReplicationHandler.java | 11 +
.../replication/MisReplicationHandler.java | 188 ++
.../replication/OverReplicatedProcessor.java | 94 +-
.../replication/RatisMisReplicationHandler.java | 78 +
.../replication/RatisOverReplicationHandler.java | 247 ++
.../replication/RatisUnderReplicationHandler.java | 242 ++
.../container/replication/ReplicationManager.java | 73 +-
.../container/replication/ReplicationQueue.java | 17 -
.../replication/UnderReplicatedProcessor.java | 95 +-
...sor.java => UnhealthyReplicationProcessor.java} | 79 +-
.../ClosedWithMismatchedReplicasHandler.java | 64 +-
.../health/ClosedWithUnhealthyReplicasHandler.java | 4 +-
.../health/ClosingContainerHandler.java | 7 +
.../health/DeletingContainerHandler.java | 5 +
.../health/ECReplicationCheckHandler.java | 18 +-
.../replication/health/EmptyContainerHandler.java | 6 +-
.../replication/health/OpenContainerHandler.java | 11 +-
.../health/QuasiClosedContainerHandler.java | 4 +
.../health/RatisReplicationCheckHandler.java | 14 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 28 +-
.../pipeline/PipelinePlacementPolicyFactory.java | 52 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 34 +-
...inerLocationProtocolServerSideTranslatorPB.java | 17 +
.../hdds/scm/server/SCMClientProtocolServer.java | 14 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 41 +-
.../hdds/scm/server/StorageContainerManager.java | 34 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 32 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 275 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../algorithms/TestContainerPlacementFactory.java | 13 +-
.../TestContainerPlacementStatusDefault.java | 22 +-
.../TestSCMContainerPlacementRackScatter.java | 96 +-
.../TestContainerReplicaPendingOps.java | 77 +
.../replication/TestECMisReplicationHandler.java | 174 ++
.../replication/TestECUnderReplicationHandler.java | 87 +
.../replication/TestMisReplicationHandler.java | 182 ++
.../TestRatisMisReplicationHandler.java | 166 ++
.../TestRatisOverReplicationHandler.java | 281 ++
.../TestRatisUnderReplicationHandler.java | 228 ++
.../replication/TestReplicationManager.java | 114 +
.../TestClosedWithMismatchedReplicasHandler.java | 16 +-
.../health/TestECReplicationCheckHandler.java | 6 +-
.../health/TestRatisReplicationCheckHandler.java | 5 +-
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 215 ++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 15 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 22 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 14 +-
.../ozone/container/common/TestEndPoint.java | 8 +
hadoop-hdds/test-utils/pom.xml | 5 +-
.../org/apache/ozone/test/GenericTestUtils.java | 10 +
hadoop-hdds/tools/pom.xml | 8 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 1 +
.../hadoop/hdds/scm/cli/cert/CleanExpired.java | 117 +
.../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 100 +
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 58 +
.../client/io/BlockDataStreamOutputEntry.java | 287 +++
.../client/io/BlockDataStreamOutputEntryPool.java | 290 +++
.../client/io/ECBlockOutputStreamEntryPool.java | 4 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 228 +-
.../ozone/client/io/KeyDataStreamOutput.java | 510 ++++
.../hadoop/ozone/client/io/KeyInputStream.java | 436 +---
.../ozone/client/io/OzoneCryptoInputStream.java | 9 +-
.../ozone/client/io/OzoneDataStreamOutput.java | 70 +
.../ozone/client/protocol/ClientProtocol.java | 38 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 151 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 51 +-
hadoop-ozone/common/pom.xml | 4 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 7 +-
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 30 +
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/coverage.sh | 6 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 20 +
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
hadoop-ozone/dist/pom.xml | 13 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 5 +
.../dist/src/main/compose/ozone/docker-config | 2 +
.../src/main/compose/ozonesecure/docker-config | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 160 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
.../main/license/bin/licenses/LICENSE-angular.txt | 2 +-
.../src/main/license/bin/licenses/LICENSE-d3.txt | 2 +-
...{LICENSE-angular.txt => LICENSE-glyphicons.txt} | 4 +-
.../main/license/bin/licenses/LICENSE-guava.txt | 13 +
.../LICENSE-javax.activation-activation.txt | 3 -
.../LICENSE-net.sf.jopt-simple-jopt-simple.txt | 24 -
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 -
.../license/bin/licenses/LICENSE-org.aspectj.txt | 279 ++
.../bin/licenses/LICENSE-org.ow2.asm-asm.txt | 1 -
.../license/bin/licenses/LICENSE-org.slf4j.txt | 2 +-
...y-misc.txt => NOTICE-ratis-thirdparty-misc.txt} | 24 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
.../main/license/src/licenses/LICENSE-angular.txt | 2 +-
.../licenses/LICENSE-bootstrap.txt} | 4 +-
.../src/main/license/src/licenses/LICENSE-d3.txt | 4 +-
.../licenses/LICENSE-glyphicons.txt} | 4 +-
.../main/license/src/licenses/LICENSE-guava.txt | 13 +
.../main/license/src/licenses/LICENSE-jquery.txt | 2 +-
.../src/main/license/src/licenses/LICENSE-nvd3.txt | 17 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 8 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/httpfsgateway/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 4 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 81 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 158 ++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 42 +-
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 54 +
.../hdds/scm/storage/TestContainerCommandsEC.java | 4 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 20 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 21 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +-
.../hadoop/ozone/TestStorageContainerManager.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 136 +-
.../client/rpc/TestBlockDataStreamOutput.java | 273 ++
.../rpc/TestContainerStateMachineStream.java | 219 ++
.../ozone/client/rpc/TestECKeyOutputStream.java | 22 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 6 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 441 ++--
.../client/rpc/TestOzoneRpcClientAbstract.java | 101 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 129 +
.../client/rpc/read/TestChunkInputStream.java | 6 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 2 +
.../apache/hadoop/ozone/container/TestHelper.java | 42 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 187 +-
.../server/TestSecureContainerServer.java | 4 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 80 +-
.../org/apache/hadoop/ozone/om/TestLDBCli.java | 15 +
.../ozone/om/TestObjectStoreWithLegacyFS.java | 52 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 171 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 13 +
.../ozone/parser/TestOzoneHARatisLogParser.java | 61 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 36 +
.../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +
hadoop-ozone/interface-client/pom.xml | 4 +-
.../interface-client/src/main/resources/proto.lock | 2672 ++++++++++++++++----
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 10 +
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 72 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 109 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 4 +
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 25 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 8 +-
.../request/file/OMFileCreateRequestWithFSO.java | 3 +-
.../om/request/key/OMAllocateBlockRequest.java | 3 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 9 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 19 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../S3MultipartUploadCommitPartRequest.java | 8 +-
.../S3MultipartUploadCompleteRequest.java | 3 +-
.../response/file/OMDirectoryCreateResponse.java | 9 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 10 +-
.../response/file/OMFileCreateResponseWithFSO.java | 6 +
.../key/OMDirectoriesPurgeResponseWithFSO.java | 4 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 7 +
.../response/key/OMKeyDeleteResponseWithFSO.java | 4 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 4 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 12 +-
.../ozone/om/service/DirectoryDeletingService.java | 149 +-
.../webapps/ozoneManager/om-overview.html | 4 +
.../apache/hadoop/ozone/om/TestChunkStreams.java | 150 +-
.../ozone/om/request/OMRequestTestUtils.java | 12 +
.../bucket/TestOMBucketSetPropertyRequest.java | 88 +
.../request/file/TestOMDirectoryCreateRequest.java | 37 +
.../file/TestOMDirectoryCreateRequestWithFSO.java | 39 +
.../om/request/file/TestOMFileCreateRequest.java | 34 +-
.../file/TestOMFileCreateRequestWithFSO.java | 31 +
.../TestOMDirectoriesPurgeRequestAndResponse.java | 103 +-
.../om/request/key/TestOMKeyCreateRequest.java | 32 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 6 +
.../TestS3MultipartUploadCompleteRequest.java | 6 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 6 +
.../file/TestOMDirectoryCreateResponse.java | 19 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 19 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 7 +-
.../s3/multipart/TestS3MultipartResponse.java | 12 +-
.../TestS3MultipartUploadAbortResponse.java | 3 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 12 +-
hadoop-ozone/ozonefs-common/pom.xml | 8 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 27 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 36 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 47 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 20 +-
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 9 +-
hadoop-ozone/recon-codegen/pom.xml | 3 +-
hadoop-ozone/recon/pom.xml | 4 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 12 +
.../ozone/recon/api/ClusterStateEndpoint.java | 10 +-
.../ozone/recon/api/handlers/BucketHandler.java | 26 -
.../ozone/recon/api/handlers/FSOBucketHandler.java | 4 +-
.../ozone/recon/api/handlers/KeyEntityHandler.java | 3 +-
.../recon/api/handlers/LegacyBucketHandler.java | 4 +-
.../recon/api/handlers/RootEntityHandler.java | 24 +-
.../recon/api/types/ClusterStateResponse.java | 18 +
.../ozone/recon/fsck/ContainerHealthTask.java | 67 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 33 +-
.../recon/scm/ReconDatanodeProtocolServer.java | 2 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 12 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../ozone/recon/scm/ReconStaleNodeHandler.java | 58 +
.../scm/ReconStorageContainerManagerFacade.java | 180 +-
.../recon/spi/StorageContainerServiceProvider.java | 21 +
.../impl/StorageContainerServiceProviderImpl.java | 15 +
.../resources/webapps/recon/ozone-recon-web/NOTICE | 5 -
.../webapps/recon/ozone-recon-web/api/db.json | 93 +-
.../recon/ozone-recon-web/src/utils/themeIcons.tsx | 6 +-
.../src/views/datanodes/datanodes.tsx | 10 +-
.../src/views/overview/overview.tsx | 13 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 4 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 99 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 98 +-
.../recon/api/TestTotalOpenContainerCount.java | 396 +++
.../ozone/recon/fsck/TestContainerHealthTask.java | 11 +-
.../TestContainerHealthTaskRecordGenerator.java | 42 +-
.../scm/AbstractReconContainerManagerTest.java | 4 +-
hadoop-ozone/s3gateway/pom.xml | 5 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 24 +
hadoop-ozone/tools/pom.xml | 4 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 31 +-
.../ozone/debug/container/ExportSubcommand.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 14 +-
.../hadoop/ozone/freon/ContentGenerator.java | 18 +
.../ozone/freon/OzoneClientKeyGenerator.java | 29 +-
.../java/org/apache/hadoop/ozone/shell/Shell.java | 21 +-
.../ozone/shell/bucket/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 67 +-
.../ozone/shell/volume/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/volume/SetQuotaHandler.java | 11 +-
pom.xml | 154 +-
420 files changed, 20102 insertions(+), 3973 deletions(-)
diff --cc hadoop-ozone/dist/pom.xml
index 7afd0e7448,659c39c3d1..459cb6f5bd
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@@ -222,11 -222,6 +222,10 @@@
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-insight</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-httpfsgateway</artifactId>
- <version>1.3.0-SNAPSHOT</version>
</dependency>
</dependencies>
<profiles>
diff --cc hadoop-ozone/dist/src/main/license/jar-report.txt
index c53a764a34,c2561d8899..291dbd8e8d
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@@ -276,4 -265,4 +276,5 @@@ share/ozone/lib/token-provider.ja
share/ozone/lib/txw2.jar
share/ozone/lib/weld-servlet.Final.jar
share/ozone/lib/woodstox-core.jar
+share/ozone/lib/zookeeper.jar
+ share/ozone/lib/zstd-jni.jar
diff --cc hadoop-ozone/httpfsgateway/pom.xml
index 650192a9ab,0000000000..a7861cf86c
mode 100644,000000..100644
--- a/hadoop-ozone/httpfsgateway/pom.xml
+++ b/hadoop-ozone/httpfsgateway/pom.xml
@@@ -1,316 -1,0 +1,316 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone</artifactId>
- <version>1.3.0-SNAPSHOT</version>
++ <version>1.4.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>ozone-httpfsgateway</artifactId>
- <version>1.3.0-SNAPSHOT</version>
++ <version>1.4.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>Apache Ozone HttpFS</name>
+ <description>Apache Ozone HttpFS</description>
+
+ <properties>
+ <httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
+ <httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
+ <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
+ <httpfs.build.timestamp>${maven.build.timestamp}</httpfs.build.timestamp>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-filesystem-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-filesystem</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1.1</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>apache-curator</artifactId>
+ <version>2.4.0</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>2.3.0</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>httpfs.properties</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>false</filtering>
+ <excludes>
+ <exclude>httpfs.properties</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ <filtering>false</filtering>
+ </testResource>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ <filtering>true</filtering>
+ </testResource>
+ </testResources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <configuration>
+ <includeTestSourceDirectory>false</includeTestSourceDirectory>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!-- workaround for filtered/unfiltered resources in same directory -->
+ <!-- remove when maven-eclipse-plugin 2.9 is available -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
+ <threadCount>1</threadCount>
+ <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>javadoc-no-fork</goal>
+ </goals>
+ <phase>site</phase>
+ <configuration>
+ <quiet>true</quiet>
+ <verbose>false</verbose>
+ <source>${maven.compile.source}</source>
+ <charset>${maven.compile.encoding}</charset>
+ <groups>
+ <group>
+ <title>HttpFs API</title>
+ <packages>*</packages>
+ </group>
+ </groups>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>create-web-xmls</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${project.build.directory}/test-classes/webapp"/>
+
+ <copy todir="${project.build.directory}/test-classes/webapp">
+ <fileset dir="${basedir}/src/main/webapp"/>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>site</id>
+ <phase>site</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <xslt in="${basedir}/src/main/resources/httpfs-default.xml"
+ out="${project.build.directory}/site/httpfs-default.html"
+ style="${basedir}/src/site/configuration.xsl"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptorRefs>
+ <descriptorRef>hadoop-httpfs-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --cc hadoop-ozone/pom.xml
index 6d643bc855,c3aefb4afc..73196476f7
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@@ -236,6 -235,6 +236,11 @@@
<artifactId>ozone-insight</artifactId>
<version>${hdds.version}</version>
</dependency>
++ <dependency>
++ <groupId>org.apache.ozone</groupId>
++ <artifactId>ozone-httpfsgateway</artifactId>
++ <version>${ozone.version}</version>
++ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-recon</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org