You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/09/01 12:12:17 UTC
[hadoop-ozone] branch HDDS-4119 updated (b8d1e3d -> c4144cc)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch HDDS-4119
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from b8d1e3d HDDS-4099. No Log4j 2 configuration file found error appears in CLI (#1318)
add 16347b7 teragenfix
add 61a8e6a revert genesis changes
add 2e32191 cleanup patch
add 962cfd5 Cleanup tests and block output stream
add d4456c0 fix buffer pool allocation
add 1c4f272 unit test fix
add 65a542f additional debug log
add 256db2a fix write(byte) with the help of Lokesh
add 118d8ce Additional fixes from Lokesh
add 514f711 rat and checkstyle fixes
add dd99deb checkstyle fixes
add a2d8fc5 Address review comments
add ad9c07c checkstyle fixes
add 9ab01a7 move conditions to the helper methods
add 8969b42 restore orginal writeChunk logic in handleFlush
add 7bf5b29 Use incremental chunk buffer for time being
add 40721a1 Revert single writeChunk() call with different condition
add bc5b38b Merge remote-tracking branch 'elek/HDDS-4119' into HDDS-4119
add e5e89e0 HDDS-4114. Bump log4j2 version (#1325)
add 59fc0bb HDDS-4127. Components with web interface should depend on hdds-docs. (#1335)
add 1abbfed HDDS-4094. Support byte-level write in Freon HadoopFsGenerator (#1310)
add 1c7003e HDDS-4139. Update version number in upgrade tests (#1347)
add c656feb HDDS-4144. Update version info in hadoop client dependency readme (#1348)
add 122eac5 HDDS-4074. [OFS] Implement AbstractFileSystem for RootedOzoneFileSystem (#1330)
add 854fdc4 HDDS-4112. Improve SCM webui page performance (#1323)
add c0084a1 HDDS-3654. Let backgroundCreator create pipeline for the support replication factors alternately (#984)
add a2080cf HDDS-4111. Keep the CSI.zh.md consistent with CSI.md (#1320)
add 8102ac7 HDDS-4062. Non rack aware pipelines should not be created if multiple racks are alive. (#1291)
add 9292b39 HDDS-4068. Client should not retry same OM on network connection failure (#1324)
add 7f674fd HDDS-3972. Add option to limit number of items displaying through ldb tool. (#1206)
add bc7786a HDDS-4056. Convert OzoneAdmin to pluggable model (#1285)
add 5fab834 HDDS-4152. Archive container logs for kubernetes check (#1355)
add 5523636 HDDS-4140. Auto-close /pending pull requests after 21 days of inactivity (#1344)
add dcb1c6e HDDS-2411. add a datanode chunk validator fo datanode chunk generator (#1312)
add 2f3edd9 HDDS-4153. Increase default timeout in kubernetes tests (#1357)
add da61c4a HDDS-4149. Implement OzoneFileStatus#toString (#1356)
add d064230 HDDS-4109. Tests in TestOzoneFileSystem should use the existing MiniOzoneCluster (#1316)
add f6e4417 HDDS-4145. Bump version to 1.1.0-SNAPSHOT on master (#1349)
add 02289ce HDDS-4146. Show the ScmId and ClusterId in the scm web ui. (#1350)
add f64bc6e HDDS-4137. Turn on the verbose mode of safe mode check on testlib (#1343)
add 44acf78 HDDS-4147. Add OFS to FileSystem META-INF (#1352)
add 8e98977 HDDS-4151. Skip the inputstream while offset larger than zero in s3g (#1354)
add d34ab29 HDDS-3903. OzoneRpcClient support batch rename keys. (#1150)
add 78ca8bf HDDS-4077. Incomplete OzoneFileSystem statistics (#1329)
add 0ec1a8a HDDS-3867. Extend the chunkinfo tool to display information from all nodes in the pipeline. (#1154)
add 0bce14d Merge remote-tracking branch 'origin/master' into HDDS-4119
add c4144cc fix merge problem
No new revisions were added by this update.
Summary of changes:
.github/close-pending.sh | 41 ++++
.github/closing-message.txt | 7 +
.github/comment-commands/close.sh | 10 +-
.github/comment-commands/pending.sh | 1 +
.../{comments.yaml => close-pending.yaml} | 19 +-
hadoop-hdds/client/pom.xml | 15 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 30 +++
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 131 ++++++----
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 47 ++--
.../hadoop/hdds/scm/storage/CommitWatcher.java | 37 +--
.../storage/TestBlockOutputStreamCorrectness.java | 224 +++++++++++++++++
.../hadoop/hdds/scm/storage/TestBufferPool.java} | 29 ++-
hadoop-hdds/common/pom.xml | 4 +-
.../org/apache/hadoop/hdds/cli/package-info.java | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 15 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 110 +++++----
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/common/ChunkBuffer.java | 14 +-
.../common/ChunkBufferImplWithByteBuffer.java | 10 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 29 ++-
.../hadoop/ozone/common/TestChunkBuffer.java | 16 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 11 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 13 +-
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/framework/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-client/README.md | 18 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
hadoop-hdds/interface-client/pom.xml | 4 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 5 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 33 ++-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 35 +++
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 4 +
.../hdds/scm/server/StorageContainerManager.java | 8 +
.../main/resources/webapps/scm/scm-overview.html | 18 +-
.../src/main/resources/webapps/scm/scm.js | 4 -
.../hadoop/hdds/scm/container/MockNodeManager.java | 13 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 81 ++++++
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 12 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 67 +++++
.../org/apache/hadoop/hdds/cli}/package-info.java | 4 +-
.../hdds/scm/cli/ReplicationManagerCommands.java | 23 +-
.../scm/cli/ReplicationManagerStartSubcommand.java | 21 +-
.../cli/ReplicationManagerStatusSubcommand.java | 32 +--
.../scm/cli/ReplicationManagerStopSubcommand.java | 25 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 ++-
.../hadoop/hdds/scm/cli/SafeModeCommands.java | 27 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 22 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 72 ++++++
.../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 24 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 65 +++--
.../hdds/scm/cli/container/CloseSubcommand.java | 20 +-
.../hdds/scm/cli/container/ContainerCommands.java | 21 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 26 +-
.../hdds/scm/cli/container/DeleteSubcommand.java | 20 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 40 ++-
.../hdds/scm/cli/container/ListSubcommand.java | 32 +--
.../hdds/scm/cli/container/WithScmClient.java | 29 ---
.../hdds/scm/cli/datanode/DatanodeCommands.java | 21 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 48 ++--
.../cli/pipeline/ActivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 38 ++-
.../cli/pipeline/DeactivatePipelineSubcommand.java | 19 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 40 ++-
.../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +-
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 10 +
.../hadoop/ozone/client/io/OzoneInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 12 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 13 +
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../hadoop/ozone/om/exceptions/OMException.java | 5 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 16 +-
.../{OmDeleteKeys.java => OmRenameKeys.java} | 28 ++-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 18 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 9 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +++
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +-
hadoop-ozone/dist/pom.xml | 4 +-
.../main/compose/ozone-mr/hadoop27/docker-config | 1 +
.../main/compose/ozone-mr/hadoop31/docker-config | 1 +
.../main/compose/ozone-mr/hadoop32/docker-config | 1 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 1 -
.../src/main/compose/ozone-om-ha/docker-config | 1 -
.../dist/src/main/compose/ozone/docker-config | 2 -
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../main/compose/ozonesecure-om-ha/docker-config | 1 -
.../src/main/compose/ozonesecure/docker-config | 2 -
hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +-
.../dist/src/main/compose/upgrade/README.md | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +-
.../src/main/compose/upgrade/versions/README.md | 2 +-
.../main/compose/upgrade/versions/ozone-0.5.0.sh | 1 +
.../versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} | 1 +
.../src/main/k8s/examples/getting-started/test.sh | 2 +
.../dist/src/main/k8s/examples/minikube/test.sh | 2 +
.../dist/src/main/k8s/examples/ozone-dev/test.sh | 2 +
.../dist/src/main/k8s/examples/ozone/test.sh | 2 +
.../dist/src/main/k8s/examples/test-all.sh | 11 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
.../{topology/cli.robot => admincli/admin.robot} | 23 +-
.../src/main/smoketest/admincli/container.robot | 68 ++++++
.../src/main/smoketest/admincli/datanode.robot | 19 +-
.../src/main/smoketest/admincli/pipeline.robot | 49 +++-
.../smoketest/admincli/replicationmanager.robot | 53 ++++
.../src/main/smoketest/admincli/safemode.robot | 45 ++++
.../src/main/smoketest/debug/ozone-debug.robot | 4 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 2 +-
.../dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} | 2 +-
.../upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh | 0
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 15 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 22 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 9 +-
.../ozone/contract/rooted/RootedOzoneContract.java | 7 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 31 ++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 106 +++++++-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 120 +++++++++
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 235 ------------------
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 3 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OmClientProtocol.proto | 26 ++
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++++++++++++++++++++
...nameResponse.java => OMKeysRenameResponse.java} | 55 +++--
.../om/request/key/TestOMKeysRenameRequest.java | 160 ++++++++++++
.../om/response/key/TestOMKeysRenameResponse.java | 131 ++++++++++
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 71 ++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 71 ++++++
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
.../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
hadoop-ozone/pom.xml | 4 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 2 +-
hadoop-ozone/s3gateway/pom.xml | 9 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +-
hadoop-ozone/tools/pom.xml | 6 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 122 ----------
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 2 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 149 ++++++-----
.../hadoop/ozone/debug/ContainerChunkInfo.java | 21 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 62 +++--
.../org/apache/hadoop/ozone/debug/RDBParser.java | 4 +
.../hadoop/ozone/freon/ContentGenerator.java | 31 ++-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 244 +++++++++++++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 1 +
.../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 +
.../hadoop/ozone/freon/TestContentGenerator.java | 82 +++++++
.../TestGenerateOzoneRequiredConfigurations.java | 5 +-
pom.xml | 14 +-
183 files changed, 3337 insertions(+), 1333 deletions(-)
create mode 100755 .github/close-pending.sh
create mode 100644 .github/closing-message.txt
copy .github/workflows/{comments.yaml => close-pending.yaml} (75%)
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockOutputStreamCorrectness.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/scm/storage/CheckedBiFunction.java => client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBufferPool.java} (58%)
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java
copy hadoop-hdds/{test-utils/src/main/java/org/apache/hadoop/test => tools/src/main/java/org/apache/hadoop/hdds/cli}/package-info.java (92%)
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java
copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java => hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java (61%)
delete mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/{OmDeleteKeys.java => OmRenameKeys.java} (61%)
rename hadoop-ozone/dist/src/main/compose/upgrade/versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} (91%)
copy hadoop-ozone/dist/src/main/smoketest/{topology/cli.robot => admincli/admin.robot} (58%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot
rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} (94%)
rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh (100%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmLDBCli.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/{OMKeyRenameResponse.java => OMKeysRenameResponse.java} (56%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java
copy hadoop-ozone/{ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java => ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (81%)
copy hadoop-ozone/{ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java => ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (80%)
copy hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} (80%)
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org