You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ah...@apache.org on 2025/02/25 16:41:03 UTC
(hadoop) branch feature-HADOOP-19363-analytics-accelerator-s3 updated: AAL ITest updates
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-19363-analytics-accelerator-s3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/feature-HADOOP-19363-analytics-accelerator-s3 by this push:
new bae4ed47bd5 AAL ITest updates
bae4ed47bd5 is described below
commit bae4ed47bd5a3ad6a27667fe0833fba58454756b
Author: Ahmar Suhail <ah...@amazon.co.uk>
AuthorDate: Tue Feb 25 16:40:09 2025 +0000
AAL ITest updates
---
.../ITestS3AAnalyticsAcceleratorStreamReading.java | 31 +++++++++++++++-------
1 file changed, 21 insertions(+), 10 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
index 1aa8cb1f23a..0afde9352ee 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
@@ -40,7 +40,10 @@
import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration;
import software.amazon.s3.analyticsaccelerator.util.PrefetchMode;
-import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.fs.s3a.Constants.ANALYTICS_ACCELERATOR_CONFIGURATION_PREFIX;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.enableAnalyticsAccelerator;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
@@ -80,19 +83,18 @@ public Configuration createConfiguration() {
}
@Test
- public void testConnectorFrameWorkIntegration() throws IOException {
+ public void testConnectorFrameWorkIntegration() throws Throwable {
describe("Verify S3 connector framework integration");
- Configuration conf = new Configuration(getConfiguration());
- removeBaseAndBucketOverrides(conf, INPUT_FADVISE);
- conf.set(INPUT_FADVISE, "whole-file");
-
S3AFileSystem fs =
- (S3AFileSystem) FileSystem.get(externalTestFile.toUri(), conf);
+ (S3AFileSystem) FileSystem.get(externalTestFile.toUri(), getConfiguration());
byte[] buffer = new byte[500];
IOStatistics ioStats;
- try (FSDataInputStream inputStream = fs.open(externalTestFile)) {
+ try (FSDataInputStream inputStream =
+ fs.openFile(externalTestFile)
+ .must(FS_OPTION_OPENFILE_READ_POLICY, FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+ .build().get()) {
ioStats = inputStream.getIOStatistics();
inputStream.seek(5);
inputStream.read(buffer, 0, 500);
@@ -142,7 +144,7 @@ public void testMalformedParquetFooter() throws IOException {
* @throws IOException any IO problem
*/
@Test
- public void testMultiRowGroupParquet() throws IOException {
+ public void testMultiRowGroupParquet() throws Throwable {
describe("A parquet file is read successfully");
Path dest = path("multi_row_group.parquet");
@@ -162,6 +164,15 @@ public void testMultiRowGroupParquet() throws IOException {
}
verifyStatisticCounterValue(ioStats, STREAM_READ_ANALYTICS_OPENED, 1);
+
+ try (FSDataInputStream inputStream = getFileSystem().openFile(dest)
+ .must(FS_OPTION_OPENFILE_READ_POLICY,FS_OPTION_OPENFILE_READ_POLICY_PARQUET)
+ .build().get()) {
+ ioStats = inputStream.getIOStatistics();
+ inputStream.readFully(buffer, 0, (int) fileStatus.getLen());
+ }
+
+ verifyStatisticCounterValue(ioStats, STREAM_READ_ANALYTICS_OPENED, 1);
}
@Test
@@ -197,7 +208,7 @@ public void testConnectorFrameworkConfigurable() {
public void testInvalidConfigurationThrows() throws Exception {
describe("Verify S3 connector framework throws with invalid configuration");
- Configuration conf = getConfiguration();
+ Configuration conf = new Configuration(getConfiguration()) ;
removeBaseAndBucketOverrides(conf);
//Disable Sequential Prefetching
conf.setInt(ANALYTICS_ACCELERATOR_CONFIGURATION_PREFIX +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org