You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2020/03/27 02:44:17 UTC
[hadoop-ozone] branch HDDS-2823 updated (eaf0676 -> 07fcb79)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from eaf0676 HDDS-1995. Generate renewTime on OMLeader for GetDelegationToken (#676)
add 6a66efa HDDS-3181. Intermittent failure in TestReconWithOzoneManager due to BindException (#682)
add d80eba3 HDDS-3154. Intermittent failure in Test2WayCommitInRatis. (#672)
add 2ea0b60 HDDS-3086. Failure running integration test it-freon. (#671)
add fd1aba8 HDDS-3150. Implement getIfExist in Table and use it in CreateKey/File (#654)
add 14f4a29 HDDS-3153. Create REST API to serve Recon Dashboard and integrate with UI in Recon. (#681)
add 3b5bfd3 HDDS-3170. Fix issues in File count by size task. (#675)
add 053af29 HDDS-3177. Periodic dependency update (Java) (#680)
add bfb71bf HDDS-3180. Datanode fails to start due to confused inconsistent volume state. (#679)
add 752452f HDDS-3184. Fix MiniOzoneChaosTest to set the correct defaults. (#685)
add c0b18c3 HDDS-3174. Rebalance integration tests (#677)
add 73e25be HDDS-3144. LogSubcommandTest fails
add f952542 HDDS-3175. Healthy datanodes are marked as stale (#674)
add efe08a2 HDDS-3042. Support running full Ratis pipeline from IDE (IntelliJ) (#571)
add f66d6e1 HDDS-3183. Deprecate old Recon HTTP Server Keytab config key. (#689)
add 7a48111 HDDS-3220. Filesystem client should not retry on AccessControlException. (#693)
add 281faf3 HDDS-3142. Create isolated enviornment for OM to test it without SCM. (#656)
add 56def9f HDDS-2941. file create : create key table entries for intermediate directories in the path (#623)
add c535793 HDDS-3098. TestDeleteWithSlowFollower is failing intermittently. (#686)
add f0a06bc HDDS-2995. Add integration test for Recon's Passive SCM state. (#688)
add c8b6fff HDDS-3064. Get Key is hung when READ delay is injected in chunk file path. (#673)
add 5b6fc74 HDDS-2797. beyond/RunningWithHDFS.md translation (#524)
add 0201fc6 HDDS-3137. OM RpcClient fail with java.lang.IllegalArgumentException. (#695)
add 0c1bc56 HDDS-3221. Refactor SafeModeHandler to use a Notification Interface (#690)
add c64d86f HDDS-3234. Fix retry interval default in Ozone client. (#698)
add f2e259b HDDS-3235.Change to default of max retry count for Ozone client (#699)
add ad7cd6a HDDS-2878. Refactor MiniOzoneLoadGenerator to add more load generators to chaos testing. (#438)
add 512d607 Revert "HDDS-3142. Create isolated enviornment for OM to test it without SCM. (#656)"
add 8249385 HDDS-3250. Create a separate log file for Warnings and Errors in MiniOzoneChaosCluster. (#711)
add f6be766 HDDS-3243. Recon should not have the ability to send Create/Close Container commands to Datanode. (#712)
add 4682bab HDDS-3164. Add Recon endpoint to serve missing containers and its metadata. (#714)
add 07fcb79 HDDS-3284. ozonesecure-mr test fails due to lack of disk space (#725)
No new revisions were added by this update.
Summary of changes:
.github/workflows/post-commit.yml | 16 +-
.github/workflows/pr.yml | 16 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 9 +-
.../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +-
.../common/src/main/resources/ozone-default.xml | 22 +-
.../common/statemachine/StateContext.java | 40 +--
.../common/states/datanode/InitDatanodeState.java | 4 +-
.../states/endpoint/HeartbeatEndpointTask.java | 8 +-
.../states/endpoint/RegisterEndpointTask.java | 59 ++--
.../ozone/container/common/volume/HddsVolume.java | 6 +
.../common/statemachine/TestStateContext.java | 9 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 6 +-
hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 2 +-
.../docs/content/beyond/RunningWithHDFS.zh.md | 64 ++++
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 28 ++
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 26 ++
.../org/apache/hadoop/hdds/utils/db/Table.java | 17 ++
.../apache/hadoop/hdds/utils/db/TypedTable.java | 24 ++
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 63 ++++
.../hdds/utils/db/TestTypedRDBTableStore.java | 18 ++
.../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 7 +-
.../hdds/scm/container/ReplicationManager.java | 12 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 11 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 16 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 3 -
.../hadoop/hdds/scm/safemode/SafeModeHandler.java | 68 ++---
.../SafeModeNotification.java} | 23 +-
.../scm/server/OzoneStorageContainerManager.java | 4 +
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 31 --
.../hdds/scm/server/StorageContainerManager.java | 6 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hdds/scm/safemode/TestSafeModeHandler.java | 6 +-
.../scm/server/TestSCMClientProtocolServer.java | 6 +-
hadoop-hdds/tools/pom.xml | 6 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 -
.../ozone/om/ha/OMFailoverProxyProvider.java | 18 +-
.../org/apache/hadoop/ozone/om/ha/OMProxyInfo.java | 25 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 25 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 10 +-
.../{Datanode.xml => Datanode1.xml} | 6 +-
.../{Datanode.xml => Datanode2.xml} | 6 +-
.../{Datanode.xml => Datanode3.xml} | 6 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 12 +-
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 7 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 19 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 265 ++++------------
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 33 +-
.../ozone/loadgenerators/AgedLoadGenerator.java | 98 ++++++
.../hadoop/ozone/loadgenerators/DataBuffer.java | 53 ++++
.../loadgenerators/FilesystemLoadGenerator.java | 69 +++++
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 101 ++++++
.../ozone/loadgenerators/LoadGenerator.java} | 24 +-
.../ozone/loadgenerators/RandomLoadGenerator.java | 68 +++++
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 23 +-
.../src/test/resources/log4j.properties | 10 +-
.../hadoop/ozone/insight/TestLogSubcommand.java | 14 +-
hadoop-ozone/integration-test/pom.xml | 113 -------
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 70 +++--
.../hadoop/fs/ozone/contract/OzoneContract.java | 23 ++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 26 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 46 ++-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 7 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 53 ++++
.../ozone/client/rpc/Test2WayCommitInRatis.java | 10 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 4 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 1 +
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../hadoop/ozone/freon/TestDataValidate.java | 24 ++
.../freon/TestDataValidateWithDummyContainers.java | 1 +
.../ozone/freon/TestFreonWithDatanodeRestart.java | 35 ++-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 21 ++
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 200 ++++++++++++
.../ozone/recon/TestReconWithOzoneManager.java | 17 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 5 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 15 -
.../om/request/file/OMDirectoryCreateRequest.java | 69 +++--
.../ozone/om/request/file/OMFileCreateRequest.java | 143 +++------
.../ozone/om/request/file/OMFileRequest.java | 51 +++-
.../ozone/om/request/key/OMKeyCommitRequest.java | 3 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 5 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 4 +-
.../response/file/OMDirectoryCreateResponse.java | 31 +-
.../om/response/file/OMFileCreateResponse.java | 7 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 29 +-
.../om/request/file/TestOMFileCreateRequest.java | 40 ++-
.../file/TestOMDirectoryCreateResponse.java | 2 +-
.../om/response/key/TestOMKeyCreateResponse.java | 4 +-
hadoop-ozone/ozonefs-lib-legacy/pom.xml | 5 +-
.../hadoop/ozone/recon/ConfigurationProvider.java | 11 +
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +-
.../recon/ReconGuiceServletContextListener.java | 2 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 2 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 2 +-
.../hadoop/ozone/recon/ReconSchemaManager.java | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 7 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 2 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 110 +++++++
...ainerKeyService.java => ContainerEndpoint.java} | 56 +++-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/TaskStatusService.java | 2 +-
.../hadoop/ozone/recon/api/UtilizationService.java | 2 +-
.../hadoop/ozone/recon/api/package-info.java | 2 +-
.../recon/api/types/ClusterStateResponse.java | 198 ++++++++++++
.../ozone/recon/api/types/ContainerKeyPrefix.java | 2 +-
.../ozone/recon/api/types/ContainerMetadata.java | 2 +-
.../ozone/recon/api/types/ContainersResponse.java | 2 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 2 +-
.../ozone/recon/api/types/DatanodePipeline.java | 2 +-
.../recon/api/types/DatanodeStorageReport.java | 2 +-
.../ozone/recon/api/types/DatanodesResponse.java | 2 +-
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 2 +-
.../hadoop/ozone/recon/api/types/KeysResponse.java | 2 +-
...Metadata.java => MissingContainerMetadata.java} | 46 ++-
...esponse.java => MissingContainersResponse.java} | 31 +-
.../ozone/recon/api/types/PipelineMetadata.java | 41 ++-
.../ozone/recon/api/types/PipelinesResponse.java | 2 +-
.../hadoop/ozone/recon/api/types/package-info.java | 2 +-
.../ozone/recon/fsck/MissingContainerInfo.java | 2 +-
.../ozone/recon/fsck/MissingContainerTask.java | 12 +-
.../hadoop/ozone/recon/fsck/package-info.java | 2 +-
.../apache/hadoop/ozone/recon/package-info.java | 2 +-
.../recon/persistence/DataSourceConfiguration.java | 5 +-
.../persistence/DefaultDataSourceProvider.java | 5 +-
.../recon/persistence/JooqPersistenceModule.java | 2 +-
.../TransactionalMethodInterceptor.java | 2 +-
.../ozone/recon/persistence/package-info.java | 2 +-
.../recon/recovery/ReconOMMetadataManager.java | 2 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +-
.../hadoop/ozone/recon/recovery/package-info.java | 2 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 2 +-
.../ozone/recon/scm/ReconContainerManager.java | 2 +-
.../recon/scm/ReconContainerReportHandler.java | 2 +-
.../recon/scm/ReconDatanodeProtocolServer.java | 2 +-
.../ReconIncrementalContainerReportHandler.java | 2 +-
.../ozone/recon/scm/ReconNewNodeHandler.java | 2 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 25 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 2 +-
.../ozone/recon/scm/ReconPipelineManager.java | 2 +-
.../recon/scm/ReconPipelineReportHandler.java | 2 +-
.../ozone/recon/scm/ReconPolicyProvider.java | 2 +-
.../ozone/recon/scm/ReconSafeModeManager.java | 2 +-
.../hadoop/ozone/recon/scm/ReconScmTask.java | 2 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 2 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../hadoop/ozone/recon/scm/package-info.java | 2 +-
.../recon/spi/ContainerDBServiceProvider.java | 11 +-
.../recon/spi/HddsDatanodeServiceProvider.java | 5 +-
.../recon/spi/OzoneManagerServiceProvider.java | 5 +-
.../recon/spi/StorageContainerServiceProvider.java | 5 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 15 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +-
.../recon/spi/impl/ReconContainerDBProvider.java | 2 +-
.../impl/StorageContainerServiceProviderImpl.java | 2 +-
.../hadoop/ozone/recon/spi/impl/package-info.java | 2 +-
.../hadoop/ozone/recon/spi/package-info.java | 2 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 13 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 66 ++--
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 38 ++-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 52 +++-
.../ozone/recon/tasks/OMUpdateEventBatch.java | 2 +-
.../hadoop/ozone/recon/tasks/ReconOmTask.java | 2 +-
.../ozone/recon/tasks/ReconTaskController.java | 2 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 2 +-
.../hadoop/ozone/recon/tasks/package-info.java | 2 +-
.../webapps/recon/ozone-recon-web/api/db.json | 20 +-
.../webapps/recon/ozone-recon-web/package.json | 3 +-
.../webapps/recon/ozone-recon-web/src/App.tsx | 2 +-
.../src/components/OverviewCard/OverviewCard.less | 13 +-
.../src/components/OverviewCard/OverviewCard.tsx | 17 +-
.../StorageBar/StorageBar.less} | 27 +-
.../src/components/StorageBar/StorageBar.tsx | 70 +++++
.../ozone-recon-web/src/makeRouteWithSubRoutes.tsx | 4 +-
.../webapps/recon/ozone-recon-web/src/routes.tsx | 2 +-
.../ozone-recon-web/src/types/datanode.types.tsx} | 11 +-
.../src/{ => types}/routes.types.tsx | 0
.../src/views/Datanodes/Datanodes.less | 29 --
.../src/views/Datanodes/Datanodes.tsx | 41 +--
.../src/views/Overview/Overview.tsx | 86 +++---
.../ozone/recon/AbstractOMMetadataManagerTest.java | 4 +-
...rKeyService.java => TestContainerEndpoint.java} | 173 ++++++++---
.../hadoop/ozone/recon/api/TestEndpoints.java | 340 +++++++++++++--------
.../ozone/recon/fsck/TestMissingContainerTask.java | 36 ++-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 125 +++++++-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 108 ++++---
.../recon/types/GuiceInjectorUtilsForTests.java | 24 +-
pom.xml | 201 ++++++------
199 files changed, 3295 insertions(+), 1578 deletions(-)
create mode 100644 hadoop-hdds/docs/content/beyond/RunningWithHDFS.zh.md
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/{command/package-info.java => safemode/SafeModeNotification.java} (54%)
copy hadoop-ozone/dev-support/intellij/runConfigurations/{Datanode.xml => Datanode1.xml} (86%)
copy hadoop-ozone/dev-support/intellij/runConfigurations/{Datanode.xml => Datanode2.xml} (80%)
rename hadoop-ozone/dev-support/intellij/runConfigurations/{Datanode.xml => Datanode3.xml} (80%)
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
copy hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java => fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java} (66%)
create mode 100644 hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java
rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/{ContainerKeyService.java => ContainerEndpoint.java} (80%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{ContainerMetadata.java => MissingContainerMetadata.java} (55%)
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{DatanodesResponse.java => MissingContainersResponse.java} (65%)
copy hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{views/Datanodes/Datanodes.less => components/StorageBar/StorageBar.less} (73%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.tsx
copy hadoop-ozone/{common/src/main/java/org/apache/hadoop/ozone/security/acl/IOzoneObj.java => recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/datanode.types.tsx} (81%)
rename hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/{ => types}/routes.types.tsx (100%)
rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/{TestContainerKeyService.java => TestContainerEndpoint.java} (67%)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org