You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/04/27 21:12:55 UTC
[ozone] branch HDDS-4440-s3-performance updated (db7f691855 -> 240002559e)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
from db7f691855 HDDS-6651. Restore S3 Gateway config key check in TestOzoneConfigurationFields (#3354)
add db778fce96 HDDS-6457. Add freon command to run r/w mix workload using FS (#3266)
add c6314b987c HDDS-6512. Install Hugo during build if needed (#3287)
add f575e238f8 HDDS-6553. Incorrect timeout when checking balancer iteration result. (#3272)
add 9d771eb6ed HDDS-6647. Tag some known flaky tests (#3349)
add b4e8e7ac8d HDDS-6635. ReportSubCommand should provide a JSON output option (#3344)
add 73d4876dfe Revert "HDDS-6579. Add commandline argument for Container Info command. (#3306)" (#3336)
add f49a691b41 HDDS-6626. Exclude Hadoop Thirdparty from shaded Ozone FS jars (#3330)
add 4476a5be19 HDDS-6632. ReplicationManagerReport should handle unknown metric in protobuf (#3335)
add d9a623ad72 HDDS-6598. Add a BackgroundPipelineScrubber to scrub all pipelines. (#3337)
add 1944b2cda9 HDDS-6624. Make ECBlockReconstructedStripeInputStream available for datanodes (#3353)
new 240002559e Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hadoop-hdds/client/pom.xml | 4 +
.../ozone/client/io/BadDataLocationException.java | 0
.../ozone/client/io/BlockInputStreamFactory.java | 4 +-
.../client/io/BlockInputStreamFactoryImpl.java | 4 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 12 +-
.../ozone/client/io/ECBlockInputStreamFactory.java | 4 +-
.../client/io/ECBlockInputStreamFactoryImpl.java | 4 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 6 +-
.../client/io/ECBlockReconstructedInputStream.java | 0
.../io/ECBlockReconstructedStripeInputStream.java | 4 +-
.../client/io/InsufficientLocationsException.java | 0
.../hadoop/ozone/client/io/package-info.java | 0
hadoop-hdds/common/pom.xml | 4 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +
.../scm/container/ReplicationManagerReport.java | 44 ++-
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 ++-----
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../common/src/main/resources/ozone-default.xml | 9 +
.../container/TestReplicationManagerReport.java | 83 ++++++
hadoop-hdds/docs/pom.xml | 12 +-
hadoop-hdds/pom.xml | 6 +
.../scm/container/balancer/ContainerBalancer.java | 125 ++++----
.../balancer/ContainerBalancerMetrics.java | 70 +++--
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 172 +++++++++++
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 3 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 30 +-
.../container/balancer/TestContainerBalancer.java | 46 +++
.../hdds/scm/pipeline/MockPipelineManager.java | 3 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 99 +++++++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 39 ++-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 1 +
.../hdds/scm/cli/container/InfoSubcommand.java | 7 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 2 +-
.../scm/cli/container/TestReportSubCommand.java | 24 ++
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 ++++----------
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 4 +
hadoop-ozone/dev-support/checks/build.sh | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 3 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +
.../ozone/client/rpc/read/ECStreamTestUtil.java | 10 +-
.../client/rpc/read/TestECBlockInputStream.java | 36 +--
.../rpc/read/TestECBlockInputStreamProxy.java | 24 +-
.../read/TestECBlockReconstructedInputStream.java | 4 +-
.../TestECBlockReconstructedStripeInputStream.java | 26 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++++++++++++++++++++
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 3 +
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 278 ++++++++++++++++++
52 files changed, 1450 insertions(+), 456 deletions(-)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java (100%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java (95%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java (97%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java (97%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java (95%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java (97%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java (97%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java (100%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java (99%)
rename {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java (100%)
copy {hadoop-ozone => hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java (100%)
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockLocationInfo.java (54%)
rename hadoop-hdds/{framework => common}/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java (100%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteFileOps.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 240002559e68df0f1922ebda27688369f1e63c4e
Merge: db7f691855 1944b2cda9
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Wed Apr 27 17:46:51 2022 +0200
Merge remote-tracking branch 'origin/master' into HDDS-4440-s3-performance
hadoop-hdds/client/pom.xml | 4 +
.../ozone/client/io/BadDataLocationException.java | 0
.../ozone/client/io/BlockInputStreamFactory.java | 4 +-
.../client/io/BlockInputStreamFactoryImpl.java | 4 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 12 +-
.../ozone/client/io/ECBlockInputStreamFactory.java | 4 +-
.../client/io/ECBlockInputStreamFactoryImpl.java | 4 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 6 +-
.../client/io/ECBlockReconstructedInputStream.java | 0
.../io/ECBlockReconstructedStripeInputStream.java | 4 +-
.../client/io/InsufficientLocationsException.java | 0
.../hadoop/ozone/client/io/package-info.java | 24 +-
hadoop-hdds/common/pom.xml | 4 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +
.../scm/container/ReplicationManagerReport.java | 44 ++-
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 ++-----
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../common/src/main/resources/ozone-default.xml | 9 +
.../container/TestReplicationManagerReport.java | 83 ++++++
hadoop-hdds/docs/pom.xml | 12 +-
hadoop-hdds/pom.xml | 6 +
.../scm/container/balancer/ContainerBalancer.java | 125 ++++----
.../balancer/ContainerBalancerMetrics.java | 70 +++--
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 172 +++++++++++
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 3 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 30 +-
.../container/balancer/TestContainerBalancer.java | 46 +++
.../hdds/scm/pipeline/MockPipelineManager.java | 3 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 99 +++++++
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 39 ++-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 1 +
.../hdds/scm/cli/container/InfoSubcommand.java | 7 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 2 +-
.../scm/cli/container/TestReportSubCommand.java | 24 ++
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 ++++----------
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 4 +
hadoop-ozone/dev-support/checks/build.sh | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 3 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +
.../ozone/client/rpc/read/ECStreamTestUtil.java | 10 +-
.../client/rpc/read/TestECBlockInputStream.java | 36 +--
.../rpc/read/TestECBlockInputStreamProxy.java | 24 +-
.../read/TestECBlockReconstructedInputStream.java | 4 +-
.../TestECBlockReconstructedStripeInputStream.java | 26 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++++++++++++++++++++
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 3 +
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 278 ++++++++++++++++++
52 files changed, 1452 insertions(+), 478 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org