You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2023/07/26 10:32:59 UTC
[hadoop] branch branch-3.3 updated: HADOOP-18805. S3A prefetch tests to work with small files (#5851)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new f5685ff639e HADOOP-18805. S3A prefetch tests to work with small files (#5851)
f5685ff639e is described below
commit f5685ff639e13d1ecabfa830848f0f3a68c76472
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Mon Jul 24 12:36:57 2023 -0600
HADOOP-18805. S3A prefetch tests to work with small files (#5851)
Contributed by Viraj Jasani
---
.../fs/s3a/ITestS3APrefetchingInputStream.java | 79 +++++++++-------------
.../fs/s3a/ITestS3APrefetchingLruEviction.java | 76 ++++++++-------------
2 files changed, 61 insertions(+), 94 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
index 2146e17b6ad..a7b59bb5d46 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import java.net.URI;
-
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,7 +34,6 @@ import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.test.LambdaTestUtils;
-import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_DEFAULT_SIZE;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMaximum;
@@ -49,7 +46,6 @@ import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_A
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE;
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_OPENED;
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS;
-import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Test the prefetching input stream, validates that the underlying S3ACachingInputStream and
@@ -64,47 +60,39 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
private static final Logger LOG =
LoggerFactory.getLogger(ITestS3APrefetchingInputStream.class);
- private static final int S_1K = 1024;
+ private static final int S_500 = 512;
+ private static final int S_1K = S_500 * 2;
private static final int S_1M = S_1K * S_1K;
- // Path for file which should have length > block size so S3ACachingInputStream is used
- private Path largeFile;
- private FileSystem largeFileFS;
private int numBlocks;
- private int blockSize;
+
+ // Size should be > block size so S3ACachingInputStream is used
private long largeFileSize;
+
// Size should be < block size so S3AInMemoryInputStream is used
- private static final int SMALL_FILE_SIZE = S_1K * 16;
+ private static final int SMALL_FILE_SIZE = S_1K * 9;
private static final int TIMEOUT_MILLIS = 5000;
private static final int INTERVAL_MILLIS = 500;
-
+ private static final int BLOCK_SIZE = S_1K * 10;
@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+ S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_BLOCK_SIZE_KEY);
conf.setBoolean(PREFETCH_ENABLED_KEY, true);
+ conf.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);
return conf;
}
- @Override
- public void teardown() throws Exception {
- super.teardown();
- cleanupWithLogger(LOG, largeFileFS);
- largeFileFS = null;
- }
-
- private void openFS() throws Exception {
- Configuration conf = getConfiguration();
- String largeFileUri = S3ATestUtils.getCSVTestFile(conf);
-
- largeFile = new Path(largeFileUri);
- blockSize = conf.getInt(PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE);
- largeFileFS = new S3AFileSystem();
- largeFileFS.initialize(new URI(largeFileUri), getConfiguration());
+ private void createLargeFile() throws Exception {
+ byte[] data = ContractTestUtils.dataset(S_1K * 72, 'x', 26);
+ Path largeFile = methodPath();
+ FileSystem largeFileFS = getFileSystem();
+ ContractTestUtils.writeDataset(getFileSystem(), largeFile, data, data.length, 16, true);
FileStatus fileStatus = largeFileFS.getFileStatus(largeFile);
largeFileSize = fileStatus.getLen();
- numBlocks = calculateNumBlocks(largeFileSize, blockSize);
+ numBlocks = calculateNumBlocks(largeFileSize, BLOCK_SIZE);
}
private static int calculateNumBlocks(long largeFileSize, int blockSize) {
@@ -119,9 +107,9 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
public void testReadLargeFileFully() throws Throwable {
describe("read a large file fully, uses S3ACachingInputStream");
IOStatistics ioStats;
- openFS();
+ createLargeFile();
- try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+ try (FSDataInputStream in = getFileSystem().open(methodPath())) {
ioStats = in.getIOStatistics();
byte[] buffer = new byte[S_1M * 10];
@@ -152,9 +140,9 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
describe("read a large file using readFully(position,buffer,offset,length),"
+ " uses S3ACachingInputStream");
IOStatistics ioStats;
- openFS();
+ createLargeFile();
- try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+ try (FSDataInputStream in = getFileSystem().open(methodPath())) {
ioStats = in.getIOStatistics();
byte[] buffer = new byte[S_1M * 10];
@@ -183,25 +171,25 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
public void testRandomReadLargeFile() throws Throwable {
describe("random read on a large file, uses S3ACachingInputStream");
IOStatistics ioStats;
- openFS();
+ createLargeFile();
- try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+ try (FSDataInputStream in = getFileSystem().open(methodPath())) {
ioStats = in.getIOStatistics();
- byte[] buffer = new byte[blockSize];
+ byte[] buffer = new byte[BLOCK_SIZE];
// Don't read block 0 completely so it gets cached on read after seek
- in.read(buffer, 0, blockSize - S_1K * 10);
+ in.read(buffer, 0, BLOCK_SIZE - S_500 * 10);
// Seek to block 2 and read all of it
- in.seek(blockSize * 2);
- in.read(buffer, 0, blockSize);
+ in.seek(BLOCK_SIZE * 2);
+ in.read(buffer, 0, BLOCK_SIZE);
// Seek to block 4 but don't read: noop.
- in.seek(blockSize * 4);
+ in.seek(BLOCK_SIZE * 4);
// Backwards seek, will use cached block 0
- in.seek(S_1K * 5);
+ in.seek(S_500 * 5);
in.read();
// Expected to get block 0 (partially read), 1 (prefetch), 2 (fully read), 3 (prefetch)
@@ -234,9 +222,9 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
byte[] buffer = new byte[SMALL_FILE_SIZE];
- in.read(buffer, 0, S_1K * 4);
- in.seek(S_1K * 12);
- in.read(buffer, 0, S_1K * 4);
+ in.read(buffer, 0, S_1K * 2);
+ in.seek(S_1K * 7);
+ in.read(buffer, 0, S_1K * 2);
verifyStatisticCounterValue(ioStats, ACTION_HTTP_GET_REQUEST, 1);
verifyStatisticCounterValue(ioStats, STREAM_READ_OPENED, 1);
@@ -261,9 +249,9 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
FSDataInputStream in = getFileSystem().open(smallFile);
byte[] buffer = new byte[SMALL_FILE_SIZE];
- in.read(buffer, 0, S_1K * 4);
- in.seek(S_1K * 12);
- in.read(buffer, 0, S_1K * 4);
+ in.read(buffer, 0, S_1K * 2);
+ in.seek(S_1K * 7);
+ in.read(buffer, 0, S_1K * 2);
long pos = in.getPos();
IOStatistics ioStats = in.getIOStatistics();
@@ -298,7 +286,6 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
inputStreamStatistics, newInputStreamStatistics);
assertFalse("seekToNewSource() not supported with prefetch", in.seekToNewSource(10));
-
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java
index bbe01887588..a8211e24ce7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.s3a;
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.CountDownLatch;
@@ -37,19 +36,17 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.test.LambdaTestUtils;
-import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_DEFAULT_SIZE;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_MAX_BLOCKS_COUNT;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticGaugeValue;
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE;
-import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* Test the prefetching input stream with LRU cache eviction on S3ACachingInputStream.
@@ -63,9 +60,7 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
public static Collection<Object[]> params() {
return Arrays.asList(new Object[][]{
{"1"},
- {"2"},
- {"3"},
- {"4"}
+ {"2"}
});
}
@@ -78,45 +73,32 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
LoggerFactory.getLogger(ITestS3APrefetchingLruEviction.class);
private static final int S_1K = 1024;
- // Path for file which should have length > block size so S3ACachingInputStream is used
- private Path largeFile;
- private FileSystem largeFileFS;
- private int blockSize;
+ private static final int S_500 = 512;
+ private static final int SMALL_FILE_SIZE = S_1K * 56;
- private static final int TIMEOUT_MILLIS = 5000;
+ private static final int TIMEOUT_MILLIS = 3000;
private static final int INTERVAL_MILLIS = 500;
+ private static final int BLOCK_SIZE = S_1K * 10;
@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_MAX_BLOCKS_COUNT);
+ S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_BLOCK_SIZE_KEY);
conf.setBoolean(PREFETCH_ENABLED_KEY, true);
conf.setInt(PREFETCH_MAX_BLOCKS_COUNT, Integer.parseInt(maxBlocks));
+ conf.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);
return conf;
}
- @Override
- public void teardown() throws Exception {
- super.teardown();
- cleanupWithLogger(LOG, largeFileFS);
- largeFileFS = null;
- }
-
- private void openFS() throws Exception {
- Configuration conf = getConfiguration();
- String largeFileUri = S3ATestUtils.getCSVTestFile(conf);
-
- largeFile = new Path(largeFileUri);
- blockSize = conf.getInt(PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE);
- largeFileFS = new S3AFileSystem();
- largeFileFS.initialize(new URI(largeFileUri), getConfiguration());
- }
-
@Test
public void testSeeksWithLruEviction() throws Throwable {
IOStatistics ioStats;
- openFS();
+ byte[] data = ContractTestUtils.dataset(SMALL_FILE_SIZE, 'x', 26);
+ // Path for file which should have length > block size so S3ACachingInputStream is used
+ Path smallFile = methodPath();
+ ContractTestUtils.writeDataset(getFileSystem(), smallFile, data, data.length, 16, true);
ExecutorService executorService = Executors.newFixedThreadPool(5,
new ThreadFactoryBuilder()
@@ -125,7 +107,7 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
.build());
CountDownLatch countDownLatch = new CountDownLatch(7);
- try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+ try (FSDataInputStream in = getFileSystem().open(methodPath())) {
ioStats = in.getIOStatistics();
// tests to add multiple blocks in the prefetch cache
// and let LRU eviction take place as more cache entries
@@ -135,43 +117,43 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
executorService.submit(() -> readFullyWithPositionedRead(countDownLatch,
in,
0,
- blockSize - S_1K * 10));
+ BLOCK_SIZE - S_500 * 10));
// Seek to block 1 and don't read completely
executorService.submit(() -> readFullyWithPositionedRead(countDownLatch,
in,
- blockSize,
- 2 * S_1K));
+ BLOCK_SIZE,
+ 2 * S_500));
// Seek to block 2 and don't read completely
executorService.submit(() -> readFullyWithSeek(countDownLatch,
in,
- blockSize * 2L,
- 2 * S_1K));
+ BLOCK_SIZE * 2L,
+ 2 * S_500));
// Seek to block 3 and don't read completely
executorService.submit(() -> readFullyWithPositionedRead(countDownLatch,
in,
- blockSize * 3L,
- 2 * S_1K));
+ BLOCK_SIZE * 3L,
+ 2 * S_500));
// Seek to block 4 and don't read completely
executorService.submit(() -> readFullyWithSeek(countDownLatch,
in,
- blockSize * 4L,
- 2 * S_1K));
+ BLOCK_SIZE * 4L,
+ 2 * S_500));
// Seek to block 5 and don't read completely
executorService.submit(() -> readFullyWithPositionedRead(countDownLatch,
in,
- blockSize * 5L,
- 2 * S_1K));
+ BLOCK_SIZE * 5L,
+ 2 * S_500));
// backward seek, can't use block 0 as it is evicted
executorService.submit(() -> readFullyWithSeek(countDownLatch,
in,
- S_1K * 5,
- 2 * S_1K));
+ S_500 * 5,
+ 2 * S_500));
countDownLatch.await();
@@ -205,8 +187,7 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
*/
private boolean readFullyWithPositionedRead(CountDownLatch countDownLatch, FSDataInputStream in,
long position, int len) {
- byte[] buffer = new byte[blockSize];
- // Don't read block 0 completely
+ byte[] buffer = new byte[BLOCK_SIZE];
try {
in.readFully(position, buffer, 0, len);
countDownLatch.countDown();
@@ -228,8 +209,7 @@ public class ITestS3APrefetchingLruEviction extends AbstractS3ACostTest {
*/
private boolean readFullyWithSeek(CountDownLatch countDownLatch, FSDataInputStream in,
long position, int len) {
- byte[] buffer = new byte[blockSize];
- // Don't read block 0 completely
+ byte[] buffer = new byte[BLOCK_SIZE];
try {
in.seek(position);
in.readFully(buffer, 0, len);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org