You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2021/06/02 13:06:40 UTC
[ozone] 03/03: Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f904539c0fa0798f0fbc4c09e5ae98bf06d1a91e
Merge: c960ec7 69b92a0
Author: Elek Márton <el...@apache.org>
AuthorDate: Wed Jun 2 11:35:59 2021 +0200
Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
.github/workflows/post-commit.yml | 26 +-
hadoop-hdds/client/pom.xml | 30 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 29 +-
.../hadoop/hdds/scm/storage/ByteArrayReader.java | 67 ++++
.../hadoop/hdds/scm/storage/ByteBufferReader.java | 71 +++++
.../hdds/scm/storage/ByteReaderStrategy.java | 41 +++
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 38 ++-
.../hdds/scm/storage/TestBlockInputStream.java | 35 +++
hadoop-hdds/common/pom.xml | 26 +-
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 5 -
.../java/org/apache/hadoop/hdds/HddsUtils.java | 17 +
hadoop-hdds/config/pom.xml | 10 +-
hadoop-hdds/container-service/pom.xml | 42 +--
.../container/common/report/ReportPublisher.java | 4 +-
.../common/statemachine/DatanodeConfiguration.java | 30 ++
.../common/statemachine/DatanodeStateMachine.java | 6 +
.../CloseContainerCommandHandler.java | 9 +-
.../states/endpoint/HeartbeatEndpointTask.java | 4 +
.../container/common/volume/HddsVolumeChecker.java | 11 +-
.../container/common/volume/MutableVolumeSet.java | 42 ++-
.../container/keyvalue/helpers/ChunkUtils.java | 12 +
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 5 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 2 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 10 +
.../container/common/TestBlockDeletingService.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 5 +-
.../common/report/TestReportPublisher.java | 9 +-
.../statemachine/TestDatanodeConfiguration.java | 12 +
.../container/common/volume/TestVolumeSet.java | 5 +-
.../common/volume/TestVolumeSetDiskChecks.java | 11 +-
.../keyvalue/TestKeyValueBlockIterator.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 3 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 33 ++
.../container/ozoneimpl/TestContainerReader.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 3 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 68 +++-
hadoop-hdds/docs/content/feature/scm-secure-ha.png | Bin 0 -> 46757 bytes
hadoop-hdds/docs/content/interface/O3fs.md | 4 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 8 +-
hadoop-hdds/docs/content/interface/Ofs.md | 4 +-
hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 6 +-
.../docs/content/recipe/SparkOzoneFSK8S.zh.md | 6 +-
hadoop-hdds/docs/pom.xml | 6 +-
hadoop-hdds/framework/pom.xml | 34 +-
...etManager.java => ContainerTokenGenerator.java} | 58 ++--
.../token/ContainerTokenSecretManager.java | 30 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 6 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 6 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 6 +-
hadoop-hdds/interface-admin/pom.xml | 10 +-
hadoop-hdds/interface-client/pom.xml | 6 +-
hadoop-hdds/interface-server/pom.xml | 10 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
hadoop-hdds/pom.xml | 78 ++---
hadoop-hdds/server-scm/pom.xml | 56 ++--
.../scm/container/CloseContainerEventHandler.java | 9 +
.../hdds/scm/container/ReplicationManager.java | 12 +-
.../scm/container/balancer/ContainerBalancer.java | 341 ++++++++++++++++++---
.../balancer/ContainerBalancerConfiguration.java | 52 ++--
.../balancer/ContainerBalancerMetrics.java | 127 ++++++++
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 13 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 1 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 29 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 29 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 17 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 4 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 10 +
.../hdds/scm/server/StorageContainerManager.java | 52 +++-
.../main/resources/webapps/scm/scm-overview.html | 8 +
.../hadoop/hdds/scm/container/MockNodeManager.java | 88 ++++--
.../container/balancer/TestContainerBalancer.java | 219 +++++++++++++
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 6 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 4 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 25 ++
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 2 +-
hadoop-hdds/test-utils/pom.xml | 6 +-
hadoop-hdds/tools/pom.xml | 26 +-
.../hdds/scm/cli/ContainerOperationClient.java | 42 ++-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 14 +-
hadoop-ozone/client/pom.xml | 14 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 51 ++-
.../hadoop/ozone/client/io/OzoneInputStream.java | 15 +-
hadoop-ozone/common/pom.xml | 26 +-
.../hadoop/ozone/client/io/LengthInputStream.java | 17 +-
hadoop-ozone/csi/pom.xml | 14 +-
hadoop-ozone/datanode/pom.xml | 14 +-
hadoop-ozone/dev-support/checks/coverage.sh | 3 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
hadoop-ozone/dev-support/docker/Dockerfile | 2 +-
.../intellij/runConfigurations/CsiServer.xml | 4 +-
.../intellij/runConfigurations/Datanode1.xml | 2 +-
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
.../intellij/runConfigurations/FreonStandalone.xml | 4 +-
.../intellij/runConfigurations/OzoneFsShell.xml | 2 +-
.../intellij/runConfigurations/OzoneManager.xml | 2 +-
.../runConfigurations/OzoneManagerInit.xml | 2 +-
.../intellij/runConfigurations/OzoneShell.xml | 2 +-
.../intellij/runConfigurations/Recon.xml | 4 +-
.../intellij/runConfigurations/S3Gateway.xml | 4 +-
.../runConfigurations/StorageContainerManager.xml | 2 +-
.../StorageContainerManagerInit.xml | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 3 +
.../dist/dev-support/bin/dist-tar-stitching | 6 +-
hadoop-ozone/dist/pom.xml | 73 ++---
.../docker-image/docker-krb5/Dockerfile-krb5 | 35 ---
.../common/docker-image/docker-krb5/README.md | 34 --
.../common/docker-image/docker-krb5/kadm5.acl | 20 --
.../common/docker-image/docker-krb5/launcher.sh | 25 --
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../main/compose/ozone-mr/hadoop27/docker-config | 4 +-
.../main/compose/ozone-mr/hadoop31/docker-config | 4 +-
.../main/compose/ozone-mr/hadoop32/docker-config | 4 +-
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 44 ++-
.../src/main/compose/ozonesecure-ha/docker-config | 24 +-
.../compose/ozonesecure-ha/keytabs/HTTP.keytab | Bin 0 -> 144 bytes
.../main/compose/ozonesecure-ha/keytabs/dn.keytab | Bin 0 -> 278 bytes
.../main/compose/ozonesecure-ha/keytabs/om.keytab | Bin 0 -> 278 bytes
.../compose/ozonesecure-ha/keytabs/recon.keytab | Bin 0 -> 296 bytes
.../main/compose/ozonesecure-ha/keytabs/s3g.keytab | Bin 0 -> 434 bytes
.../main/compose/ozonesecure-ha/keytabs/scm.keytab | Bin 0 -> 586 bytes
.../compose/ozonesecure-ha/keytabs/testuser.keytab | Bin 0 -> 152 bytes
.../ozonesecure-ha/keytabs/testuser2.keytab | Bin 0 -> 154 bytes
.../docker-krb5 => ozonesecure-ha}/krb5.conf | 4 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../dist/src/main/compose/ozonesecure-mr/README.md | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 30 +-
.../src/main/compose/ozonesecure-mr/docker-config | 25 +-
.../compose/ozonesecure-mr/keytabs/HTTP.keytab | Bin 0 -> 144 bytes
.../main/compose/ozonesecure-mr/keytabs/dn.keytab | Bin 0 -> 278 bytes
.../main/compose/ozonesecure-mr/keytabs/om.keytab | Bin 0 -> 278 bytes
.../compose/ozonesecure-mr/keytabs/recon.keytab | Bin 0 -> 296 bytes
.../main/compose/ozonesecure-mr/keytabs/s3g.keytab | Bin 0 -> 434 bytes
.../main/compose/ozonesecure-mr/keytabs/scm.keytab | Bin 0 -> 586 bytes
.../compose/ozonesecure-mr/keytabs/testuser.keytab | Bin 0 -> 152 bytes
.../ozonesecure-mr/keytabs/testuser2.keytab | Bin 0 -> 154 bytes
.../docker-krb5 => ozonesecure-mr}/krb5.conf | 6 +-
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 23 +-
.../src/main/compose/ozonesecure/docker-config | 24 +-
.../main/compose/ozonesecure/keytabs/HTTP.keytab | Bin 0 -> 144 bytes
.../src/main/compose/ozonesecure/keytabs/dn.keytab | Bin 0 -> 278 bytes
.../src/main/compose/ozonesecure/keytabs/om.keytab | Bin 0 -> 278 bytes
.../main/compose/ozonesecure/keytabs/recon.keytab | Bin 0 -> 296 bytes
.../main/compose/ozonesecure/keytabs/s3g.keytab | Bin 0 -> 434 bytes
.../main/compose/ozonesecure/keytabs/scm.keytab | Bin 0 -> 586 bytes
.../compose/ozonesecure/keytabs/testuser.keytab | Bin 0 -> 152 bytes
.../compose/ozonesecure/keytabs/testuser2.keytab | Bin 0 -> 154 bytes
.../docker-krb5 => ozonesecure}/krb5.conf | 4 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +-
.../definitions/ozone/definitions/persistence.yaml | 4 +-
.../examples/ozone-ha/datanode-statefulset.yaml | 2 +-
.../k8s/examples/ozone/datanode-statefulset.yaml | 2 +-
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 0 -> 432 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 0 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 0 -> 146 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 0 -> 142 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 0 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 0 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 0 -> 296 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 0 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 0 -> 434 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 0 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 0 -> 450 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 0 -> 154 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 77 +++++
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 58 ++--
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 11 +-
.../dist/src/main/smoketest/basic/links.robot | 10 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 6 +-
.../dist/src/main/smoketest/cli/classpath.robot | 14 +-
.../dist/src/main/smoketest/commonlib.robot | 8 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 10 +-
.../main/smoketest/security/ozone-secure-fs.robot | 18 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 42 +--
.../fault-injection-test/mini-chaos-tests/pom.xml | 24 +-
.../fault-injection-test/network-tests/pom.xml | 6 +-
hadoop-ozone/fault-injection-test/pom.xml | 6 +-
hadoop-ozone/insight/pom.xml | 42 +--
.../hadoop/ozone/insight/BaseInsightPoint.java | 4 +-
hadoop-ozone/integration-test/pom.xml | 70 ++---
.../ozone/client/rpc/read/TestKeyInputStream.java | 63 +++-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../server/TestSecureContainerServer.java | 2 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 31 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 6 +
.../TestDatanodeHddsVolumeFailureToleration.java | 131 ++++++++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 99 ++++++
hadoop-ozone/interface-client/pom.xml | 10 +-
hadoop-ozone/interface-storage/pom.xml | 30 +-
hadoop-ozone/native-client/README.md | 4 +-
hadoop-ozone/ozone-manager/pom.xml | 60 ++--
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 24 ++
.../S3MultipartUploadCompleteRequest.java | 3 +-
hadoop-ozone/ozonefs-common/pom.xml | 42 +--
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 30 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 16 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 14 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 10 +-
hadoop-ozone/ozonefs/pom.xml | 14 +-
hadoop-ozone/pom.xml | 142 ++++-----
hadoop-ozone/recon-codegen/pom.xml | 10 +-
hadoop-ozone/recon/pom.xml | 38 +--
hadoop-ozone/s3gateway/pom.xml | 42 +--
hadoop-ozone/tools/pom.xml | 50 +--
.../ozone/debug/container/ContainerCommands.java | 2 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 +-
.../ozone/freon/ClosedContainerReplicator.java | 3 +-
.../containergenerator/GeneratorDatanode.java | 2 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/audit/parser/TestAuditParser.java | 31 +-
.../tools/src/test/resources/testaudit.log | 3 +
.../resources/{testaudit.log => testloadaudit.log} | 24 +-
pom.xml | 4 +-
224 files changed, 2973 insertions(+), 1208 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org