You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2020/09/04 05:11:25 UTC
[hadoop-ozone] branch HDDS-2939 updated (b8d1e3d -> ead6371)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a change to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from b8d1e3d HDDS-4099. No Log4j 2 configuration file found error appears in CLI (#1318)
add e5e89e0 HDDS-4114. Bump log4j2 version (#1325)
add 59fc0bb HDDS-4127. Components with web interface should depend on hdds-docs. (#1335)
add 1abbfed HDDS-4094. Support byte-level write in Freon HadoopFsGenerator (#1310)
add 1c7003e HDDS-4139. Update version number in upgrade tests (#1347)
add c656feb HDDS-4144. Update version info in hadoop client dependency readme (#1348)
add 122eac5 HDDS-4074. [OFS] Implement AbstractFileSystem for RootedOzoneFileSystem (#1330)
add 854fdc4 HDDS-4112. Improve SCM webui page performance (#1323)
add c0084a1 HDDS-3654. Let backgroundCreator create pipeline for the support replication factors alternately (#984)
add a2080cf HDDS-4111. Keep the CSI.zh.md consistent with CSI.md (#1320)
add 8102ac7 HDDS-4062. Non rack aware pipelines should not be created if multiple racks are alive. (#1291)
add 9292b39 HDDS-4068. Client should not retry same OM on network connection failure (#1324)
add 7f674fd HDDS-3972. Add option to limit number of items displaying through ldb tool. (#1206)
add bc7786a HDDS-4056. Convert OzoneAdmin to pluggable model (#1285)
add 5fab834 HDDS-4152. Archive container logs for kubernetes check (#1355)
add 5523636 HDDS-4140. Auto-close /pending pull requests after 21 days of inactivity (#1344)
add dcb1c6e HDDS-2411. add a datanode chunk validator fo datanode chunk generator (#1312)
add 2f3edd9 HDDS-4153. Increase default timeout in kubernetes tests (#1357)
add da61c4a HDDS-4149. Implement OzoneFileStatus#toString (#1356)
add d064230 HDDS-4109. Tests in TestOzoneFileSystem should use the existing MiniOzoneCluster (#1316)
add f6e4417 HDDS-4145. Bump version to 1.1.0-SNAPSHOT on master (#1349)
add 02289ce HDDS-4146. Show the ScmId and ClusterId in the scm web ui. (#1350)
add f64bc6e HDDS-4137. Turn on the verbose mode of safe mode check on testlib (#1343)
add 44acf78 HDDS-4147. Add OFS to FileSystem META-INF (#1352)
add 8e98977 HDDS-4151. Skip the inputstream while offset larger than zero in s3g (#1354)
add d34ab29 HDDS-3903. OzoneRpcClient support batch rename keys. (#1150)
add 78ca8bf HDDS-4077. Incomplete OzoneFileSystem statistics (#1329)
add 0ec1a8a HDDS-3867. Extend the chunkinfo tool to display information from all nodes in the pipeline. (#1154)
add 34ee831 HDDS-4121. Implement OmMetadataMangerImpl#getExpiredOpenKeys. (#1351)
add 13fe31b HDDS-4167. Acceptance test logs missing if fails during cluster startup (#1366)
add 9cef3f6 HDDS-4176. Fix failed UT: test2WayCommitForTimeoutException (#1370)
add 199512b HDDS-4131. Container report should update container key count and bytes used if they differ in SCM (#1339)
add 77d56e6 HDDS-4165. GitHub Actions cache does not work outside of workspace (#1364)
add c77e7ba HDDS-3804. Recon start fails with SQL exception with MySQL DB. (#1377)
add b58054e HDDS-4169. Fix some minor errors in StorageContainerManager.md (#1367)
add d2fb937 HDDS-4200. Fix missing right bracket in HA doc (#1380)
add dc5a997 HDDS-1889. Add support for verifying multiline log entry (#1308)
add 79f9fab HDDS-3762. Intermittent failure in TestDeleteWithSlowFollower (#1376)
add fd63aac HDDS-4150. Disabling flaky unit test until HDDS-4150 is fixed.
add b2fca43 HDDS-4189. Add alias `roles` for `ozone admin om` subcommand `getserviceroles` (#1375)
add c8d5334 HDDS-4199. Fix failed UT: TestOMAllocateBlockRequest#testValidateAndUpdateCache (#1379)
add ead6371 HDDS-3840. Use OFS in MapReduce acceptance test (#1365)
No new revisions were added by this update.
Summary of changes:
.github/close-pending.sh | 41 ++++
.github/closing-message.txt | 7 +
.github/comment-commands/close.sh | 10 +-
.github/comment-commands/pending.sh | 1 +
.../{comments.yaml => close-pending.yaml} | 19 +-
.github/workflows/post-commit.yml | 6 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 30 +++
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +
hadoop-hdds/common/pom.xml | 9 +-
.../org/apache/hadoop/hdds/cli/package-info.java | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 11 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 34 +++
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 62 ++++-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 11 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 2 +-
.../ozone/container/ContainerTestHelper.java | 5 +
.../content/concept/StorageContainerManager.md | 5 +-
hadoop-hdds/docs/content/feature/HA.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 13 +-
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/framework/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-client/README.md | 18 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
hadoop-hdds/interface-client/pom.xml | 4 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 5 +-
.../container/AbstractContainerReportHandler.java | 47 +++-
.../hdds/scm/container/ReplicationManager.java | 34 ---
.../scm/pipeline/BackgroundPipelineCreator.java | 33 ++-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 35 +++
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 4 +
.../hdds/scm/server/StorageContainerManager.java | 8 +
.../main/resources/webapps/scm/scm-overview.html | 18 +-
.../src/main/resources/webapps/scm/scm.js | 4 -
.../hadoop/hdds/scm/container/MockNodeManager.java | 13 +
.../scm/container/TestContainerReportHandler.java | 164 ++++++++++++-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 81 ++++++
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 12 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 67 +++++
.../org/apache/hadoop/hdds/cli}/package-info.java | 4 +-
.../hdds/scm/cli/ReplicationManagerCommands.java | 23 +-
.../scm/cli/ReplicationManagerStartSubcommand.java | 21 +-
.../cli/ReplicationManagerStatusSubcommand.java | 32 +--
.../scm/cli/ReplicationManagerStopSubcommand.java | 25 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 ++-
.../hadoop/hdds/scm/cli/SafeModeCommands.java | 27 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 22 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 72 ++++++
.../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 24 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 65 +++--
.../hdds/scm/cli/container/CloseSubcommand.java | 20 +-
.../hdds/scm/cli/container/ContainerCommands.java | 21 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 26 +-
.../hdds/scm/cli/container/DeleteSubcommand.java | 20 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 40 ++-
.../hdds/scm/cli/container/ListSubcommand.java | 32 +--
.../hdds/scm/cli/container/WithScmClient.java | 29 ---
.../hdds/scm/cli/datanode/DatanodeCommands.java | 21 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 48 ++--
.../cli/pipeline/ActivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 38 ++-
.../cli/pipeline/DeactivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 40 ++-
.../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +-
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 10 +
.../hadoop/ozone/client/io/OzoneInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 12 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 13 +
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../hadoop/ozone/om/exceptions/OMException.java | 5 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 16 +-
.../{OmDeleteKeys.java => OmRenameKeys.java} | 28 ++-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 18 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 9 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +++
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +-
hadoop-ozone/dist/pom.xml | 4 +-
.../main/compose/ozone-mr/hadoop27/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +-
.../main/compose/ozone-mr/hadoop31/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 7 +-
.../main/compose/ozone-mr/hadoop32/docker-config | 1 +
.../src/main/compose/ozone-mr/hadoop32/test.sh | 7 +-
.../dist/src/main/compose/ozone-mr/test.sh | 22 +-
.../src/main/compose/ozone-om-ha-s3/docker-config | 1 -
.../src/main/compose/ozone-om-ha/docker-config | 1 -
.../dist/src/main/compose/ozone/docker-config | 2 -
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../dist/src/main/compose/ozonesecure-mr/test.sh | 5 +-
.../main/compose/ozonesecure-om-ha/docker-config | 1 -
.../src/main/compose/ozonesecure/docker-config | 2 -
hadoop-ozone/dist/src/main/compose/test-all.sh | 21 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 40 ++-
.../dist/src/main/compose/upgrade/README.md | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +-
.../src/main/compose/upgrade/versions/README.md | 2 +-
.../main/compose/upgrade/versions/ozone-0.5.0.sh | 1 +
.../versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} | 1 +
.../src/main/k8s/examples/getting-started/test.sh | 2 +
.../dist/src/main/k8s/examples/minikube/test.sh | 2 +
.../dist/src/main/k8s/examples/ozone-dev/test.sh | 2 +
.../dist/src/main/k8s/examples/ozone/test.sh | 2 +
.../dist/src/main/k8s/examples/test-all.sh | 11 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
.../{topology/cli.robot => admincli/admin.robot} | 23 +-
.../src/main/smoketest/admincli/container.robot | 68 ++++++
.../src/main/smoketest/admincli/datanode.robot | 19 +-
.../src/main/smoketest/admincli/pipeline.robot | 49 +++-
.../smoketest/admincli/replicationmanager.robot | 53 ++++
.../src/main/smoketest/admincli/safemode.robot | 45 ++++
.../dist/src/main/smoketest/createmrenv.robot | 13 +-
.../src/main/smoketest/debug/ozone-debug.robot | 4 +-
.../dist/src/main/smoketest/mapreduce.robot | 11 +-
.../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 12 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 2 +-
.../dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} | 2 +-
.../upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh | 0
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 15 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 22 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 9 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 7 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 106 +++++++-
.../ozone/client/rpc/TestWatchForCommit.java | 5 +-
.../commandhandler/TestBlockDeletion.java | 5 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 120 +++++++++
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 235 ------------------
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 3 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OmClientProtocol.proto | 26 ++
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 11 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 5 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 10 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 35 ++-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 42 +---
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++++++++++++++++++++
...nameResponse.java => OMKeysRenameResponse.java} | 55 +++--
.../hadoop/ozone/om/TestOmMetadataManager.java | 77 ++++++
.../ozone/om/request/TestOMRequestUtils.java | 33 ++-
.../om/request/key/TestOMAllocateBlockRequest.java | 22 +-
.../om/request/key/TestOMKeysRenameRequest.java | 160 ++++++++++++
.../om/response/key/TestOMKeysRenameResponse.java | 131 ++++++++++
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 71 ++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 71 ++++++
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/pom.xml | 4 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../recon/schema/ContainerSchemaDefinition.java | 4 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 2 +-
.../ozone/recon/schema/StatsSchemaDefinition.java | 2 +-
.../recon/schema/UtilizationSchemaDefinition.java | 10 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 2 +
hadoop-ozone/s3gateway/pom.xml | 9 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +-
hadoop-ozone/tools/pom.xml | 6 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 122 ----------
.../ozone/admin/om/GetServiceRolesSubcommand.java | 4 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 2 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 149 ++++++-----
.../hadoop/ozone/debug/ContainerChunkInfo.java | 21 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 62 +++--
.../org/apache/hadoop/ozone/debug/RDBParser.java | 4 +
.../hadoop/ozone/freon/ContentGenerator.java | 31 ++-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 244 +++++++++++++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 1 +
.../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../hadoop/ozone/freon/TestContentGenerator.java | 82 +++++++
.../TestGenerateOzoneRequiredConfigurations.java | 5 +-
pom.xml | 14 +-
209 files changed, 3412 insertions(+), 1345 deletions(-)
create mode 100755 .github/close-pending.sh
create mode 100644 .github/closing-message.txt
copy .github/workflows/{comments.yaml => close-pending.yaml} (75%)
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java
copy hadoop-hdds/{test-utils/src/main/java/org/apache/hadoop/test => tools/src/main/java/org/apache/hadoop/hdds/cli}/package-info.java (92%)
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java => hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java (61%)
delete mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/{OmDeleteKeys.java => OmRenameKeys.java} (61%)
rename hadoop-ozone/dist/src/main/compose/upgrade/versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} (91%)
copy hadoop-ozone/dist/src/main/smoketest/{topology/cli.robot => admincli/admin.robot} (58%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot
rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} (94%)
rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmLDBCli.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/{OMKeyRenameResponse.java => OMKeysRenameResponse.java} (56%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java
copy hadoop-ozone/{ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java => ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (81%)
copy hadoop-ozone/{ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java => ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (80%)
copy hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} (80%)
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org