You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/08/18 12:53:21 UTC
[hadoop] branch trunk updated (cd72f7e0428 -> 682931a6ace)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from cd72f7e0428 YARN-11224. [Federation] Add getAppQueue, updateAppQueue REST APIs for Router. (#4747)
add 682931a6ace HADOOP-18028. High performance S3A input stream (#4752)
No new revisions were added by this update.
Summary of changes:
.../apache/hadoop/fs/impl/prefetch/BlockCache.java | 70 +++
.../apache/hadoop/fs/impl/prefetch/BlockData.java | 250 ++++++++
.../hadoop/fs/impl/prefetch/BlockManager.java | 145 +++++
.../hadoop/fs/impl/prefetch/BlockOperations.java | 425 ++++++++++++++
.../fs/impl/prefetch/BoundedResourcePool.java | 195 +++++++
.../apache/hadoop/fs/impl/prefetch/BufferData.java | 319 +++++++++++
.../apache/hadoop/fs/impl/prefetch/BufferPool.java | 323 +++++++++++
.../fs/impl/prefetch/CachingBlockManager.java | 638 +++++++++++++++++++++
.../impl/prefetch/EmptyPrefetchingStatistics.java | 80 +++
.../impl/prefetch/ExecutorServiceFuturePool.java | 70 +++
.../hadoop/fs/impl/prefetch/FilePosition.java | 301 ++++++++++
.../fs/impl/prefetch/PrefetchingStatistics.java | 67 +++
.../hadoop/fs/impl/prefetch/ResourcePool.java | 71 +++
.../apache/hadoop/fs/impl/prefetch/Retryer.java | 93 +++
.../fs/impl/prefetch/SingleFilePerBlockCache.java | 354 ++++++++++++
.../apache/hadoop/fs/impl/prefetch/Validate.java | 399 +++++++++++++
.../hadoop/fs/impl/prefetch/package-info.java | 28 +
.../hadoop/fs/statistics/StreamStatisticNames.java | 40 ++
.../hadoop/fs/impl/prefetch/ExceptionAsserts.java | 63 ++
.../fs/impl/prefetch/SampleDataForTests.java | 71 +++
.../hadoop/fs/impl/prefetch/TestBlockCache.java | 98 ++++
.../hadoop/fs/impl/prefetch/TestBlockData.java | 159 +++++
.../fs/impl/prefetch/TestBlockOperations.java | 106 ++++
.../fs/impl/prefetch/TestBoundedResourcePool.java | 147 +++++
.../hadoop/fs/impl/prefetch/TestBufferData.java | 244 ++++++++
.../hadoop/fs/impl/prefetch/TestBufferPool.java | 158 +++++
.../prefetch/TestExecutorServiceFuturePool.java | 98 ++++
.../hadoop/fs/impl/prefetch/TestFilePosition.java | 195 +++++++
.../hadoop/fs/impl/prefetch/TestRetryer.java | 80 +++
.../hadoop/fs/impl/prefetch/TestValidate.java | 341 +++++++++++
.../java/org/apache/hadoop/fs/s3a/Constants.java | 25 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 63 +-
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 61 +-
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 53 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 12 +
.../hadoop/fs/s3a/prefetch/S3ABlockManager.java | 78 +++
.../fs/s3a/prefetch/S3ACachingBlockManager.java | 96 ++++
.../fs/s3a/prefetch/S3ACachingInputStream.java | 214 +++++++
.../fs/s3a/prefetch/S3AInMemoryInputStream.java | 102 ++++
.../fs/s3a/prefetch/S3APrefetchingInputStream.java | 260 +++++++++
.../fs/s3a/prefetch/S3ARemoteInputStream.java | 475 +++++++++++++++
.../hadoop/fs/s3a/prefetch/S3ARemoteObject.java | 328 +++++++++++
.../fs/s3a/prefetch/S3ARemoteObjectReader.java | 175 ++++++
.../hadoop/fs/s3a/prefetch/package-info.java | 29 +
.../s3a/statistics/S3AInputStreamStatistics.java | 3 +-
.../statistics/impl/EmptyS3AStatisticsContext.java | 36 ++
.../src/site/markdown/tools/hadoop-aws/index.md | 24 +
.../site/markdown/tools/hadoop-aws/prefetching.md | 192 +++++++
.../fs/s3a/ITestS3APrefetchingInputStream.java | 189 ++++++
.../hadoop/fs/s3a/ITestS3ARequesterPays.java | 17 +-
.../org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java | 10 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 20 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 4 +-
.../fs/s3a/prefetch/MockS3ARemoteObject.java | 114 ++++
.../hadoop/fs/s3a/prefetch/S3APrefetchFakes.java | 413 +++++++++++++
.../fs/s3a/prefetch/TestS3ABlockManager.java | 87 +++
.../s3a/prefetch/TestS3ACachingBlockManager.java | 361 ++++++++++++
.../fs/s3a/prefetch/TestS3ARemoteInputStream.java | 279 +++++++++
.../fs/s3a/prefetch/TestS3ARemoteObject.java | 88 +++
.../fs/s3a/prefetch/TestS3ARemoteObjectReader.java | 112 ++++
.../s3a/scale/ITestS3AInputStreamPerformance.java | 9 +
61 files changed, 9534 insertions(+), 23 deletions(-)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org