You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/02/24 07:23:37 UTC
(hop) branch main updated: Upgrade s3 and allow multiple accounts, fixes #2333 (#6647)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 6bfe5bda2b Upgrade s3 and allow multiple accounts, fixes #2333 (#6647)
6bfe5bda2b is described below
commit 6bfe5bda2bd8e0d2013cbc2e7c90535e89f4ee21
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Tue Feb 24 08:23:32 2026 +0100
Upgrade s3 and allow multiple accounts, fixes #2333 (#6647)
---
assemblies/debug/pom.xml | 6 +
.../apache/hop/core/fileinput/FileInputList.java | 6 +-
.../core/fileinput/NonAccessibleFileObject.java | 2 +-
.../hop/metadata/api/HopMetadataPropertyType.java | 1 +
.../fileinput/NonAccessibleFileObjectTest.java | 104 ++++
lib/pom.xml | 48 +-
plugins/tech/aws/pom.xml | 46 +-
.../hop/pipeline/transforms/aws/sns/AwsSns.java | 66 +--
.../transforms/aws/sns/SnsNotifyDialog.java | 42 +-
.../pipeline/transforms/aws/sqs/AwsSqsReader.java | 99 ++--
.../hop/pipeline/transforms/aws/sqs/SqsReader.java | 12 +-
.../transforms/aws/sqs/SqsReaderDialog.java | 44 +-
.../java/org/apache/hop/vfs/s3/S3VfsPlugin.java | 18 +-
.../org/apache/hop/vfs/s3/amazon/s3/S3Util.java | 20 -
.../org/apache/hop/vfs/s3/metadata/AwsRegions.java | 62 +++
.../S3AuthType.java} | 20 +-
.../org/apache/hop/vfs/s3/metadata/S3Meta.java | 75 +++
.../apache/hop/vfs/s3/metadata/S3MetaEditor.java | 568 +++++++++++++++++++++
.../org/apache/hop/vfs/s3/s3/vfs/S3FileName.java | 24 +-
.../org/apache/hop/vfs/s3/s3/vfs/S3FileObject.java | 151 +-----
.../apache/hop/vfs/s3/s3/vfs/S3FileProvider.java | 83 ++-
.../org/apache/hop/vfs/s3/s3a/vfs/S3AFileName.java | 82 ---
.../hop/vfs/s3/s3a/vfs/S3AFileNameParser.java | 59 ---
.../apache/hop/vfs/s3/s3a/vfs/S3AFileObject.java | 28 -
.../apache/hop/vfs/s3/s3a/vfs/S3AFileProvider.java | 44 --
.../apache/hop/vfs/s3/s3a/vfs/S3AFileSystem.java | 36 --
.../vfs/s3/s3common/S3CommonFileInputStream.java | 13 +-
.../hop/vfs/s3/s3common/S3CommonFileObject.java | 466 ++++++++++-------
.../hop/vfs/s3/s3common/S3CommonFileProvider.java | 6 +
.../hop/vfs/s3/s3common/S3CommonFileSystem.java | 184 ++++---
.../s3common/S3CommonFileSystemConfigBuilder.java | 34 +-
.../vfs/s3/s3common/S3CommonPipedOutputStream.java | 168 +++---
.../vfs/s3/s3common/S3CommonWindowedSubstream.java | 34 --
.../apache/hop/vfs/s3/s3common/S3ListCache.java | 122 +++++
.../org/apache/hop/vfs/s3/s3n/vfs/S3NFileName.java | 73 ---
.../hop/vfs/s3/s3n/vfs/S3NFileNameParser.java | 59 ---
.../apache/hop/vfs/s3/s3n/vfs/S3NFileProvider.java | 44 --
.../apache/hop/vfs/s3/s3n/vfs/S3NFileSystem.java | 36 --
.../s3/metadata/messages/messages_en_US.properties | 47 ++
plugins/tech/aws/src/main/resources/s3.svg | 34 ++
.../hop/vfs/s3/s3common/S3ListCacheTest.java | 174 +++++++
.../apache/hop/vfs/s3/vfs/S3FileObjectTest.java | 340 ++++++------
.../apache/hop/vfs/s3/vfs/S3FileProviderTest.java | 128 ++++-
.../apache/hop/vfs/s3/vfs/S3FileSystemTest.java | 33 ++
.../hop/vfs/s3/vfs/S3NFileNameParserTest.java | 58 ---
.../org/apache/hop/vfs/s3/vfs/S3NFileNameTest.java | 66 ---
.../apache/hop/vfs/s3/vfs/S3NFileObjectTest.java | 350 -------------
.../apache/hop/vfs/s3/vfs/S3NFileProviderTest.java | 44 --
.../apache/hop/vfs/s3/vfs/S3NFileSystemTest.java | 70 ---
.../transforms/getfilenames/GetFileNamesMeta.java | 1 +
.../GetFileNamesMetaAdditionalTest.java | 2 +-
.../getfilenames/GetFileNamesTransformTest.java | 2 +
52 files changed, 2290 insertions(+), 2044 deletions(-)
diff --git a/assemblies/debug/pom.xml b/assemblies/debug/pom.xml
index 7ab7d7e21e..f7571832a4 100644
--- a/assemblies/debug/pom.xml
+++ b/assemblies/debug/pom.xml
@@ -403,6 +403,12 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-tech-aws</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-tech-azure</artifactId>
diff --git a/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java b/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java
index 4ac23b1876..4de6c55849 100644
--- a/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java
+++ b/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java
@@ -89,7 +89,11 @@ public class FileInputList {
public static String getRequiredFilesDescription(List<FileObject> nonExistantFiles) {
StringBuilder buffer = new StringBuilder();
for (FileObject file : nonExistantFiles) {
- buffer.append(file.getName().getURI());
+ try {
+ buffer.append(file.getName().getURI());
+ } catch (Exception e) {
+ buffer.append(file.getPublicURIString());
+ }
buffer.append(Const.CR);
}
return buffer.toString();
diff --git a/core/src/main/java/org/apache/hop/core/fileinput/NonAccessibleFileObject.java b/core/src/main/java/org/apache/hop/core/fileinput/NonAccessibleFileObject.java
index e02773c8a1..6f40faa4dc 100644
--- a/core/src/main/java/org/apache/hop/core/fileinput/NonAccessibleFileObject.java
+++ b/core/src/main/java/org/apache/hop/core/fileinput/NonAccessibleFileObject.java
@@ -131,7 +131,7 @@ public class NonAccessibleFileObject implements FileObject {
@Override
public String getPublicURIString() {
- throw new NotImplementedException();
+ return fullyQualifiedName;
}
@Override
diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java
index dc28eeed28..8808989e4f 100644
--- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java
+++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java
@@ -99,4 +99,5 @@ public enum HopMetadataPropertyType {
VFS_GCP_CONNECTION,
VFS_AZURE_CONNECTION,
VFS_MINIO_CONNECTION,
+ VFS_S3_CONNECTION,
}
diff --git a/core/src/test/java/org/apache/hop/core/fileinput/NonAccessibleFileObjectTest.java b/core/src/test/java/org/apache/hop/core/fileinput/NonAccessibleFileObjectTest.java
new file mode 100644
index 0000000000..df18913c78
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/fileinput/NonAccessibleFileObjectTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.core.fileinput;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.apache.commons.vfs2.FileSystemException;
+import org.junit.Test;
+
+public class NonAccessibleFileObjectTest {
+
+ @Test
+ public void testGetPublicURIString() {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket/mykey");
+ assertEquals("s3:///mybucket/mykey", obj.getPublicURIString());
+ }
+
+ @Test
+ public void testGetPublicURIStringWithNullPath() {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject(null);
+ assertEquals(null, obj.getPublicURIString());
+ }
+
+ @Test
+ public void testExists() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.exists());
+ }
+
+ @Test
+ public void testIsReadable() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.isReadable());
+ }
+
+ @Test
+ public void testIsWriteable() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.isWriteable());
+ }
+
+ @Test
+ public void testCanRenameTo() {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.canRenameTo(null));
+ }
+
+ @Test
+ public void testGetURL() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("http://example.com/file");
+ assertEquals("http://example.com/file", obj.getURL().toString());
+ }
+
+ @Test(expected = FileSystemException.class)
+ public void testGetURLWithInvalidURL() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("not a valid url");
+ obj.getURL();
+ }
+
+ @Test
+ public void testIsAttached() {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.isAttached());
+ }
+
+ @Test
+ public void testIsContentOpen() {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.isContentOpen());
+ }
+
+ @Test
+ public void testIsHidden() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ assertFalse(obj.isHidden());
+ }
+
+ @Test
+ public void testCloseDoesNotThrow() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ obj.close();
+ }
+
+ @Test
+ public void testRefreshDoesNotThrow() throws FileSystemException {
+ NonAccessibleFileObject obj = new NonAccessibleFileObject("s3:///mybucket");
+ obj.refresh();
+ }
+}
diff --git a/lib/pom.xml b/lib/pom.xml
index 3e55756d0a..2402fef371 100644
--- a/lib/pom.xml
+++ b/lib/pom.xml
@@ -39,7 +39,8 @@
<apache-spark.version>3.4.3</apache-spark.version>
<asm.version>9.7</asm.version>
<avro.version>1.12.0</avro.version>
- <aws-java-sdk.version>1.12.651</aws-java-sdk.version>
+ <!-- AWS: SDK v2 only (software.amazon.awssdk). Aligned with Apache Beam 2.71.0. -->
+ <aws-java-sdk.version>2.20.162</aws-java-sdk.version>
<bahir.version>2.1.1</bahir.version>
<bcprov-jdk18on.version>1.81</bcprov-jdk18on.version>
<blueprints-core.version>2.6.0</blueprints-core.version>
@@ -181,40 +182,31 @@
<artifactId>kafka-schema-registry-client</artifactId>
<version>${kafka-schema-registry-client.version}</version>
</dependency>
+ <!-- AWS SDK v2 (single version for all AWS usage in Hop) -->
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-core</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- <version>${nimbus-jose.version}</version>
- </dependency>
- <dependency>
- <groupId>com.squareup.okio</groupId>
- <artifactId>okio-jvm</artifactId>
- <version>${okio.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sns</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-emr</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sqs</artifactId>
<version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-iam</artifactId>
- <version>${aws-java-sdk.version}</version>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ <version>${nimbus-jose.version}</version>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-kms</artifactId>
- <version>${aws-java-sdk.version}</version>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio-jvm</artifactId>
+ <version>${okio.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@@ -232,16 +224,6 @@
<artifactId>commons-text</artifactId>
<version>${commons-text.version}</version>
</dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-pricing</artifactId>
- <version>${aws-java-sdk.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-redshift</artifactId>
- <version>${aws-java-sdk.version}</version>
- </dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>spark-avro_2.11</artifactId>
diff --git a/plugins/tech/aws/pom.xml b/plugins/tech/aws/pom.xml
index 53718aae05..96d64481a9 100644
--- a/plugins/tech/aws/pom.xml
+++ b/plugins/tech/aws/pom.xml
@@ -29,7 +29,8 @@
<name>Hop Plugins Technology AWS</name>
<properties>
- <aws-java-sdk.version>1.12.651</aws-java-sdk.version>
+ <!-- AWS SDK v2: aligned with Apache Beam 2.71.0 (see lib/pom.xml) -->
+ <aws-java-sdk.version>2.20.162</aws-java-sdk.version>
</properties>
<dependencyManagement>
@@ -52,51 +53,24 @@
</dependencyManagement>
<dependencies>
+ <!-- AWS SDK v2: S3, SNS, SQS (aligned with Beam 2.71.0) -->
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-core</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
<version>${aws-java-sdk.version}</version>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sns</artifactId>
<version>${aws-java-sdk.version}</version>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sns</artifactId>
- <version>${aws-java-sdk.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sqs</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sqs</artifactId>
<version>${aws-java-sdk.version}</version>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>com.amazon.redshift</groupId>
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/AwsSns.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/AwsSns.java
index 8bda872f4b..e536596eed 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/AwsSns.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/AwsSns.java
@@ -17,22 +17,21 @@
package org.apache.hop.pipeline.transforms.aws.sns;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.services.sns.AmazonSNSClient;
-import com.amazonaws.services.sns.AmazonSNSClientBuilder;
-import com.amazonaws.services.sns.model.AmazonSNSException;
-import com.amazonaws.services.sns.model.PublishRequest;
-import com.amazonaws.services.sns.model.PublishResult;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.sns.SnsClient;
+import software.amazon.awssdk.services.sns.SnsClientBuilder;
+import software.amazon.awssdk.services.sns.model.PublishRequest;
+import software.amazon.awssdk.services.sns.model.PublishResponse;
+import software.amazon.awssdk.services.sns.model.SnsException;
public class AwsSns {
- private AmazonSNSClient snsClient;
+ private SnsClient snsClient;
private SnsNotifyMeta meta;
private String awsKey;
private String awsRegion;
@@ -63,27 +62,19 @@ public class AwsSns {
try {
baseTransform.logBasic("Starting connection to AWS SNS");
- if (this.awsCredChain.equalsIgnoreCase("N")) {
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(this.awsKey, this.awsKeySecret);
- snsClient =
- (AmazonSNSClient)
- AmazonSNSClientBuilder.standard()
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
- .withRegion(this.awsRegion)
- .build();
+ SnsClientBuilder builder = SnsClient.builder().region(Region.of(this.awsRegion));
+ if (this.awsCredChain.equalsIgnoreCase("N")) {
+ AwsBasicCredentials awsCreds = AwsBasicCredentials.create(this.awsKey, this.awsKeySecret);
+ snsClient = builder.credentialsProvider(StaticCredentialsProvider.create(awsCreds)).build();
baseTransform.logBasic(
"Connected to SNS in Region "
+ this.awsRegion
+ " with API-Key >>"
+ this.awsKey
+ "<<");
-
} else {
- AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
- snsClient =
- (AmazonSNSClient) AmazonSNSClientBuilder.standard().withCredentials(provider).build();
-
+ snsClient = builder.credentialsProvider(DefaultCredentialsProvider.create()).build();
baseTransform.logBasic("Connected to SNS with provided Credentials Chain");
}
return true;
@@ -98,11 +89,11 @@ public class AwsSns {
/** Disconnects from AWS */
public void disconnectAWSConnection() {
try {
- snsClient.shutdown();
-
+ if (snsClient != null) {
+ snsClient.close();
+ }
baseTransform.logBasic("Disconnected from SNS in Region " + this.awsRegion);
-
- } catch (AmazonClientException e) {
+ } catch (Exception e) {
baseTransform.logError(e.getMessage());
baseTransform.setErrors(1);
}
@@ -116,22 +107,17 @@ public class AwsSns {
* @param msg Message Content
* @return SNS messageID on successful publish
*/
- public String publishToSNS(String tARN, String subj, String msg) throws AmazonSNSException {
+ public String publishToSNS(String tARN, String subj, String msg) throws SnsException {
String topicARN = baseTransform.resolve(tARN);
String subject = baseTransform.resolve(subj);
String message = baseTransform.resolve(msg);
- try {
-
- PublishRequest publishRequest = new PublishRequest(topicARN, message, subject);
- PublishResult publishResult = snsClient.publish(publishRequest);
- String messageId = publishResult.getMessageId();
- baseTransform.logBasic(messageId);
- return messageId;
-
- } catch (AmazonSNSException e) {
- throw e;
- }
+ PublishRequest publishRequest =
+ PublishRequest.builder().topicArn(topicARN).message(message).subject(subject).build();
+ PublishResponse publishResult = snsClient.publish(publishRequest);
+ String messageId = publishResult.messageId();
+ baseTransform.logBasic(messageId);
+ return messageId;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/SnsNotifyDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/SnsNotifyDialog.java
index 7112e7ea6e..fb3ea42b87 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/SnsNotifyDialog.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sns/SnsNotifyDialog.java
@@ -17,12 +17,7 @@
package org.apache.hop.pipeline.transforms.aws.sns;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.RegionUtils;
-import com.amazonaws.services.sns.AmazonSNS;
import java.util.Arrays;
-import java.util.List;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowMeta;
@@ -505,24 +500,35 @@ public class SnsNotifyDialog extends BaseTransformDialog {
new Thread(fieldLoader).start();
}
- /** This method fills the CombarVar with all available AWS Regions */
+ /** This method fills the ComboVar with common AWS region IDs (SNS-supported). */
private void populateAWSRegion(ComboVar tAWSRegion2) {
-
tAWSRegion2.removeAll();
-
- try {
-
- List<Region> snsRegions = RegionUtils.getRegionsForService(AmazonSNS.ENDPOINT_PREFIX);
-
- for (Region region : snsRegions) {
- tAWSRegion2.add(region.getName());
- }
-
- } catch (AmazonClientException e) {
- logError(BaseMessages.getString(PKG, e.getMessage()));
+ for (String regionId : AWS_REGION_IDS) {
+ tAWSRegion2.add(regionId);
}
}
+ private static final String[] AWS_REGION_IDS =
+ new String[] {
+ "us-east-1",
+ "us-east-2",
+ "us-west-1",
+ "us-west-2",
+ "eu-west-1",
+ "eu-west-2",
+ "eu-central-1",
+ "eu-north-1",
+ "ap-southeast-1",
+ "ap-southeast-2",
+ "ap-northeast-1",
+ "ap-northeast-2",
+ "ap-south-1",
+ "ca-central-1",
+ "sa-east-1",
+ "af-south-1",
+ "me-south-1"
+ };
+
/**
* This helper method puts the transform configuration stored in the meta object and puts it into
* the dialog controls.
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/AwsSqsReader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/AwsSqsReader.java
index 2dffec7ccd..dffbb4d20f 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/AwsSqsReader.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/AwsSqsReader.java
@@ -17,23 +17,24 @@
package org.apache.hop.pipeline.transforms.aws.sqs;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.services.sqs.AmazonSQSClient;
-import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
-import com.amazonaws.services.sqs.model.AmazonSQSException;
-import com.amazonaws.services.sqs.model.Message;
-import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import java.util.List;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.sqs.SqsClient;
+import software.amazon.awssdk.services.sqs.SqsClientBuilder;
+import software.amazon.awssdk.services.sqs.model.DeleteMessageRequest;
+import software.amazon.awssdk.services.sqs.model.Message;
+import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;
+import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;
+import software.amazon.awssdk.services.sqs.model.SqsException;
public class AwsSqsReader {
- private AmazonSQSClient sqsClient;
+ private SqsClient sqsClient;
private SqsReaderMeta meta;
private String awsKey;
private String awsRegion;
@@ -43,13 +44,6 @@ public class AwsSqsReader {
private String awsCredChain;
private String deleteMessage;
- /**
- * Constructor for new AWS SQS Object
- *
- * @param transformMeta SqsReaderMeta
- * @param t PipelineMeta
- * @param bst BaseTransform
- */
public AwsSqsReader(SqsReaderMeta transformMeta, PipelineMeta t, BaseTransform bst) {
this.meta = transformMeta;
@@ -72,20 +66,13 @@ public class AwsSqsReader {
try {
baseTransform.logBasic("Starting connection to AWS SQS");
- if (this.awsCredChain.equalsIgnoreCase("N")) {
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(this.awsKey, this.awsKeySecret);
- sqsClient =
- (AmazonSQSClient)
- AmazonSQSClientBuilder.standard()
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
- .withRegion(this.awsRegion)
- .build();
+ SqsClientBuilder builder = SqsClient.builder().region(Region.of(this.awsRegion));
+ if (this.awsCredChain.equalsIgnoreCase("N")) {
+ AwsBasicCredentials awsCreds = AwsBasicCredentials.create(this.awsKey, this.awsKeySecret);
+ sqsClient = builder.credentialsProvider(StaticCredentialsProvider.create(awsCreds)).build();
} else {
- AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
- sqsClient =
- (AmazonSQSClient) AmazonSQSClientBuilder.standard().withCredentials(provider).build();
-
+ sqsClient = builder.credentialsProvider(DefaultCredentialsProvider.create()).build();
baseTransform.logBasic("Connected to SQS with provided Credentials Chain");
}
return true;
@@ -99,48 +86,46 @@ public class AwsSqsReader {
/** Disconnects from AWS */
public void disconnectAWSConnection() {
try {
- sqsClient.shutdown();
-
+ if (sqsClient != null) {
+ sqsClient.close();
+ }
baseTransform.logBasic("Disconnected from SQS");
-
- } catch (AmazonClientException e) {
+ } catch (Exception e) {
baseTransform.logError(e.getMessage());
baseTransform.setErrors(1);
}
}
/**
- * @param queueURL
- * @param numMsgs
- * @param isPreview
- * @return
- * @throws AmazonSQSException
+ * @param queueURL queue URL
+ * @param numMsgs max number of messages
+ * @param isPreview whether this is a preview
+ * @return list of SQS messages
+ * @throws SqsException on SQS errors
*/
public List<Message> readMessages(String queueURL, int numMsgs, boolean isPreview)
- throws AmazonSQSException {
+ throws SqsException {
int numMessages = (numMsgs > 10) ? 10 : numMsgs;
- try {
-
- ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(queueURL);
- receiveMessageRequest.setMaxNumberOfMessages(numMessages);
- List<Message> messages = sqsClient.receiveMessage(receiveMessageRequest).getMessages();
+ ReceiveMessageRequest receiveMessageRequest =
+ ReceiveMessageRequest.builder().queueUrl(queueURL).maxNumberOfMessages(numMessages).build();
+ ReceiveMessageResponse response = sqsClient.receiveMessage(receiveMessageRequest);
+ List<Message> messages = response.messages();
- baseTransform.logDebug(messages.size() + " Message(s) retrieved from queue");
+ baseTransform.logDebug(messages.size() + " Message(s) retrieved from queue");
- if (this.deleteMessage.equalsIgnoreCase("Y") && !isPreview) {
-
- for (Message m : messages) {
- sqsClient.deleteMessage(queueURL, m.getReceiptHandle());
- }
- baseTransform.logDebug(messages.size() + " Message(s) deleted from queue");
+ if (this.deleteMessage.equalsIgnoreCase("Y") && !isPreview) {
+ for (Message m : messages) {
+ sqsClient.deleteMessage(
+ DeleteMessageRequest.builder()
+ .queueUrl(queueURL)
+ .receiptHandle(m.receiptHandle())
+ .build());
}
-
- return messages;
-
- } catch (AmazonSQSException e) {
- throw e;
+ baseTransform.logDebug(messages.size() + " Message(s) deleted from queue");
}
+
+ return messages;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReader.java
index 34d09eea4a..0161c55b90 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReader.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReader.java
@@ -17,7 +17,6 @@
package org.apache.hop.pipeline.transforms.aws.sqs;
-import com.amazonaws.services.sqs.model.Message;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
@@ -34,6 +33,7 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
+import software.amazon.awssdk.services.sqs.model.Message;
public class SqsReader extends BaseTransform<SqsReaderMeta, SqsReaderData> {
@@ -151,28 +151,28 @@ public class SqsReader extends BaseTransform<SqsReaderMeta, SqsReaderData> {
int idxMessageIdField = data.outputRowMeta.indexOfValue(data.realMessageIDFieldName);
if (idxMessageIdField >= 0) {
- outputRow[idxMessageIdField] = m.getMessageId();
+ outputRow[idxMessageIdField] = m.messageId();
}
int idxMessageBodyField = data.outputRowMeta.indexOfValue(data.realMessageBodyFieldName);
if (idxMessageBodyField >= 0) {
- outputRow[idxMessageBodyField] = m.getBody();
+ outputRow[idxMessageBodyField] = m.body();
}
int idxReceiptHandleField =
data.outputRowMeta.indexOfValue(data.realReceiptHandleFieldName);
if (idxReceiptHandleField >= 0) {
- outputRow[idxReceiptHandleField] = m.getReceiptHandle();
+ outputRow[idxReceiptHandleField] = m.receiptHandle();
}
int idxBodyMD5Field = data.outputRowMeta.indexOfValue(data.realBodyMD5FieldName);
if (idxBodyMD5Field >= 0) {
- outputRow[idxBodyMD5Field] = m.getMD5OfBody();
+ outputRow[idxBodyMD5Field] = m.md5OfBody();
}
int idxSNSMessageField = data.outputRowMeta.indexOfValue(data.realSNSMessageFieldName);
if (idxSNSMessageField >= 0) {
- outputRow[idxSNSMessageField] = getSNSMessageContent(m.getBody());
+ outputRow[idxSNSMessageField] = getSNSMessageContent(m.body());
}
putRow(data.outputRowMeta, outputRow);
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReaderDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReaderDialog.java
index 3a0bf609fe..52199ccbb6 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReaderDialog.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/aws/sqs/SqsReaderDialog.java
@@ -17,11 +17,6 @@
package org.apache.hop.pipeline.transforms.aws.sqs;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.RegionUtils;
-import com.amazonaws.services.sqs.AmazonSQS;
-import java.util.List;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.PipelineMeta;
@@ -591,27 +586,38 @@ public class SqsReaderDialog extends BaseTransformDialog {
}
/**
- * This method fills the CombarVar with all available AWS Regions
+ * This method fills the ComboVar with common AWS region IDs (SQS-supported).
*
- * @param ComboVar tAWSRegion2
+ * @param tAWSRegion2 ComboVar to populate
*/
private void populateAWSRegion(ComboVar tAWSRegion2) {
-
tAWSRegion2.removeAll();
-
- try {
-
- List<Region> snsRegions = RegionUtils.getRegionsForService(AmazonSQS.ENDPOINT_PREFIX);
-
- for (Region region : snsRegions) {
- tAWSRegion2.add(region.getName());
- }
-
- } catch (AmazonClientException e) {
- logError(BaseMessages.getString(PKG, e.getMessage()));
+ for (String regionId : AWS_REGION_IDS) {
+ tAWSRegion2.add(regionId);
}
}
+ private static final String[] AWS_REGION_IDS =
+ new String[] {
+ "us-east-1",
+ "us-east-2",
+ "us-west-1",
+ "us-west-2",
+ "eu-west-1",
+ "eu-west-2",
+ "eu-central-1",
+ "eu-north-1",
+ "ap-southeast-1",
+ "ap-southeast-2",
+ "ap-northeast-1",
+ "ap-northeast-2",
+ "ap-south-1",
+ "ca-central-1",
+ "sa-east-1",
+ "af-south-1",
+ "me-south-1"
+ };
+
/**
* This helper method puts the transform configuration stored in the meta object and puts it into
* the dialog controls.
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
index 126dcbb30d..a886b8bb70 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
@@ -17,11 +17,16 @@
package org.apache.hop.vfs.s3;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.commons.vfs2.provider.FileProvider;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.plugin.IVfs;
import org.apache.hop.core.vfs.plugin.VfsPlugin;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataUtil;
+import org.apache.hop.vfs.s3.metadata.S3Meta;
import org.apache.hop.vfs.s3.s3.vfs.S3FileProvider;
@VfsPlugin(type = "s3", typeDescription = "S3 VFS plugin", classLoaderGroup = "vfs-s3")
@@ -38,6 +43,17 @@ public class S3VfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
- return null;
+ Map<String, FileProvider> providers = new HashMap<>();
+ try {
+ IHopMetadataProvider metadataProvider =
+ HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ List<S3Meta> s3MetaList = metadataProvider.getSerializer(S3Meta.class).loadAll();
+ for (S3Meta s3Meta : s3MetaList) {
+ providers.put(s3Meta.getName(), new S3FileProvider(variables, s3Meta));
+ }
+ } catch (Exception e) {
+ // Ignore errors (e.g. metadata not initialized)
+ }
+ return providers;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/amazon/s3/S3Util.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/amazon/s3/S3Util.java
index 9dd3452d6e..681e61732c 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/amazon/s3/S3Util.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/amazon/s3/S3Util.java
@@ -19,30 +19,10 @@ package org.apache.hop.vfs.s3.amazon.s3;
public final class S3Util {
- /** System property name for the AWS Endpoint (This is for Minio) */
- public static final String ENDPOINT_SYSTEM_PROPERTY = "aws.endpoint";
-
- /** System property name for the AWS Signature version (This is for Minio) */
- public static final String SIGNATURE_VERSION_SYSTEM_PROPERTY = "aws.endpoint";
-
- /** System property name for the AWS access key ID */
public static final String ACCESS_KEY_SYSTEM_PROPERTY = "aws.accessKeyId";
-
- /** System property name for the AWS secret key */
public static final String SECRET_KEY_SYSTEM_PROPERTY = "aws.secretKey";
-
- /** Environment variable for the AWS region */
public static final String AWS_REGION = "AWS_REGION";
- /** Environment variable for the specific location of the AWS config file */
- public static final String AWS_CONFIG_FILE = "AWS_CONFIG_FILE";
-
- /** AWS configuration folder */
- public static final String AWS_FOLDER = ".aws";
-
- /** Configuration file name */
- public static final String CONFIG_FILE = "config";
-
public static boolean hasChanged(String previousValue, String currentValue) {
if (!isEmpty(previousValue) && isEmpty(currentValue)) {
return true;
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/AwsRegions.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/AwsRegions.java
new file mode 100644
index 0000000000..fc65692792
--- /dev/null
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/AwsRegions.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.s3.metadata;
+
+/** Standard AWS region IDs for S3 (commonly used regions). */
+public final class AwsRegions {
+
+ private static final String[] REGIONS =
+ new String[] {
+ "",
+ "af-south-1",
+ "ap-east-1",
+ "ap-northeast-1",
+ "ap-northeast-2",
+ "ap-northeast-3",
+ "ap-south-1",
+ "ap-south-2",
+ "ap-southeast-1",
+ "ap-southeast-2",
+ "ap-southeast-3",
+ "ap-southeast-4",
+ "ca-central-1",
+ "ca-west-1",
+ "eu-central-1",
+ "eu-central-2",
+ "eu-north-1",
+ "eu-south-1",
+ "eu-south-2",
+ "eu-west-1",
+ "eu-west-2",
+ "eu-west-3",
+ "il-central-1",
+ "me-central-1",
+ "me-south-1",
+ "sa-east-1",
+ "us-east-1",
+ "us-east-2",
+ "us-west-1",
+ "us-west-2",
+ };
+
+ private AwsRegions() {}
+
+ public static String[] getRegionIds() {
+ return REGIONS.clone();
+ }
+}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileObject.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3AuthType.java
similarity index 65%
rename from plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileObject.java
rename to plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3AuthType.java
index ceb97c9ccf..cbd2a00c44 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileObject.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3AuthType.java
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,14 +15,14 @@
* limitations under the License.
*/
-package org.apache.hop.vfs.s3.s3n.vfs;
+package org.apache.hop.vfs.s3.metadata;
-import org.apache.commons.vfs2.provider.AbstractFileName;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileObject;
-
-public class S3NFileObject extends S3CommonFileObject {
-
- public S3NFileObject(final AbstractFileName name, final S3NFileSystem fileSystem) {
- super(name, fileSystem);
- }
+/** S3 authentication method. */
+public enum S3AuthType {
+ /** Default credential chain (environment, instance profile, etc.) */
+ DEFAULT,
+ /** Explicit access key and secret (and optional session token) */
+ ACCESS_KEYS,
+ /** AWS credentials file with optional profile name */
+ CREDENTIALS_FILE
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
new file mode 100644
index 0000000000..ca1428fd08
--- /dev/null
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.s3.metadata;
+
+import java.io.Serializable;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadata;
+
+@Getter
+@Setter
+@GuiPlugin
+@HopMetadata(
+ key = "S3ConnectionDefinition",
+ name = "i18n::S3Meta.name",
+ description = "i18n::S3Meta.description",
+ image = "s3.svg",
+ documentationUrl = "/metadata-types/s3-connection.html",
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_S3_CONNECTION)
+public class S3Meta extends HopMetadataBase implements Serializable, IHopMetadata {
+
+ @HopMetadataProperty private String description;
+
+ @HopMetadataProperty(password = true)
+ private String accessKey;
+
+ @HopMetadataProperty(password = true)
+ private String secretKey;
+
+ @HopMetadataProperty(password = true)
+ private String sessionToken;
+
+ @HopMetadataProperty private String region;
+
+ @HopMetadataProperty private String endpoint;
+
+ @HopMetadataProperty private boolean pathStyleAccess;
+
+ @HopMetadataProperty private String credentialsFile;
+
+ @HopMetadataProperty private String profileName;
+
+ @HopMetadataProperty private String partSize;
+
+ /** Cache TTL in seconds for list-result caching (avoids redundant headObject calls). */
+ @HopMetadataProperty private String cacheTtlSeconds;
+
+ /** Authentication type: {@link S3AuthType#name()} (Default, ACCESS_KEYS, CREDENTIALS_FILE). */
+ @HopMetadataProperty private String authenticationType;
+
+ public S3Meta() {
+ partSize = "5MB";
+ cacheTtlSeconds = "5";
+ authenticationType = S3AuthType.DEFAULT.name();
+ }
+}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
new file mode 100644
index 0000000000..92a19827d4
--- /dev/null
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
@@ -0,0 +1,568 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.s3.metadata;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.metadata.MetadataEditor;
+import org.apache.hop.ui.core.metadata.MetadataManager;
+import org.apache.hop.ui.core.widget.TextVar;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Text;
+
+@GuiPlugin(description = "Editor for S3 connection metadata")
+public class S3MetaEditor extends MetadataEditor<S3Meta> {
+
+ private static final Class<?> PKG = S3MetaEditor.class;
+
+ /** Padding before the first widget and after the last widget inside each auth composite. */
+ private static final int INSIDE_PADDING = 6;
+
+ private PropsUi props;
+ private int middle;
+ private int margin;
+
+ private Text wName;
+ private TextVar wDescription;
+ private Combo wRegion;
+ private TextVar wPartSize;
+ private TextVar wCacheTtl;
+
+ private Label spacer;
+ private Combo wAuthType;
+
+ private Label wlAccessKey;
+ private TextVar wAccessKey;
+ private Label wlSecretKey;
+ private TextVar wSecretKey;
+ private Label wlSessionToken;
+ private TextVar wSessionToken;
+
+ private Label wlCredentialsFile;
+ private TextVar wCredentialsFile;
+ private Label wlProfileName;
+ private TextVar wProfileName;
+
+ private Label wlEndpoint;
+ private TextVar wEndpoint;
+ private Button wPathStyle;
+
+ private Composite wAuthComposite;
+ private Composite wKeysComposite;
+ private Composite wProfileComposite;
+ private Composite wCommonComposite;
+ private FormData fdKeysComposite;
+ private FormData fdProfileComposite;
+ private FormData fdCommonComposite;
+
+ public S3MetaEditor(HopGui hopGui, MetadataManager<S3Meta> manager, S3Meta metadata) {
+ super(hopGui, manager, metadata);
+ }
+
+ @Override
+ public void createControl(Composite parent) {
+ props = PropsUi.getInstance();
+ middle = props.getMiddlePct();
+ margin = PropsUi.getMargin();
+
+ Label wIcon = new Label(parent, SWT.RIGHT);
+ wIcon.setImage(getImage());
+ FormData fdlIcon = new FormData();
+ fdlIcon.top = new FormAttachment(0, 0);
+ fdlIcon.right = new FormAttachment(100, 0);
+ wIcon.setLayoutData(fdlIcon);
+ PropsUi.setLook(wIcon);
+
+ // Name row (same layout as DatabaseMetaEditor: label top=0, field below label, spacer below
+ // field)
+ Label wlName = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlName);
+ wlName.setText(BaseMessages.getString(PKG, "S3Meta.Name.Label"));
+ FormData fdlName = new FormData();
+ fdlName.top = new FormAttachment(0, 0);
+ fdlName.left = new FormAttachment(0, 0);
+ wlName.setLayoutData(fdlName);
+ wName = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wName);
+ FormData fdName = new FormData();
+ fdName.top = new FormAttachment(wlName, margin);
+ fdName.left = new FormAttachment(0, 0);
+ fdName.right = new FormAttachment(wIcon, -margin);
+ wName.setLayoutData(fdName);
+
+ spacer = new Label(parent, SWT.HORIZONTAL | SWT.SEPARATOR);
+ FormData fdSpacer = new FormData();
+ fdSpacer.left = new FormAttachment(0, 0);
+ fdSpacer.top = new FormAttachment(wName, 15);
+ fdSpacer.right = new FormAttachment(100, 0);
+ spacer.setLayoutData(fdSpacer);
+
+ // Content below spacer (15px gap, same as DatabaseMetaEditor)
+ Control lastControl = spacer;
+
+ // --- General: Description ---
+ Label wlDesc = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlDesc);
+ wlDesc.setText(BaseMessages.getString(PKG, "S3Meta.Description.Label"));
+ FormData fdlDesc = new FormData();
+ fdlDesc.top = new FormAttachment(lastControl, 15);
+ fdlDesc.left = new FormAttachment(0, 0);
+ fdlDesc.right = new FormAttachment(middle, -margin);
+ wlDesc.setLayoutData(fdlDesc);
+ wDescription = new TextVar(manager.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wDescription);
+ FormData fdDesc = new FormData();
+ fdDesc.top = new FormAttachment(wlDesc, 0, SWT.CENTER);
+ fdDesc.left = new FormAttachment(middle, 0);
+ fdDesc.right = new FormAttachment(100, 0);
+ wDescription.setLayoutData(fdDesc);
+ lastControl = wDescription;
+
+ // --- General: Region (dropdown + editable) ---
+ Label wlRegion = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlRegion);
+ wlRegion.setText(BaseMessages.getString(PKG, "S3Meta.Region.Label"));
+ FormData fdlRegion = new FormData();
+ fdlRegion.top = new FormAttachment(lastControl, margin);
+ fdlRegion.left = new FormAttachment(0, 0);
+ fdlRegion.right = new FormAttachment(middle, -margin);
+ wlRegion.setLayoutData(fdlRegion);
+ wRegion = new Combo(parent, SWT.BORDER | SWT.SINGLE | SWT.LEFT);
+ PropsUi.setLook(wRegion);
+ wRegion.setItems(AwsRegions.getRegionIds());
+ FormData fdRegion = new FormData();
+ fdRegion.top = new FormAttachment(wlRegion, 0, SWT.CENTER);
+ fdRegion.left = new FormAttachment(middle, 0);
+ fdRegion.right = new FormAttachment(100, 0);
+ wRegion.setLayoutData(fdRegion);
+ lastControl = wRegion;
+
+ // --- General: Part size ---
+ Label wlPartSize = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlPartSize);
+ wlPartSize.setText(BaseMessages.getString(PKG, "S3Meta.PartSize.Label"));
+ FormData fdlPartSize = new FormData();
+ fdlPartSize.top = new FormAttachment(lastControl, margin);
+ fdlPartSize.left = new FormAttachment(0, 0);
+ fdlPartSize.right = new FormAttachment(middle, -margin);
+ wlPartSize.setLayoutData(fdlPartSize);
+ wPartSize = new TextVar(manager.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wPartSize);
+ FormData fdPartSize = new FormData();
+ fdPartSize.top = new FormAttachment(wlPartSize, 0, SWT.CENTER);
+ fdPartSize.left = new FormAttachment(middle, 0);
+ fdPartSize.right = new FormAttachment(100, 0);
+ wPartSize.setLayoutData(fdPartSize);
+ lastControl = wPartSize;
+
+ // --- General: Cache TTL ---
+ Label wlCacheTtl = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlCacheTtl);
+ wlCacheTtl.setText(BaseMessages.getString(PKG, "S3Meta.CacheTtl.Label"));
+ FormData fdlCacheTtl = new FormData();
+ fdlCacheTtl.top = new FormAttachment(lastControl, margin);
+ fdlCacheTtl.left = new FormAttachment(0, 0);
+ fdlCacheTtl.right = new FormAttachment(middle, -margin);
+ wlCacheTtl.setLayoutData(fdlCacheTtl);
+ wCacheTtl = new TextVar(manager.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wCacheTtl);
+ FormData fdCacheTtl = new FormData();
+ fdCacheTtl.top = new FormAttachment(wlCacheTtl, 0, SWT.CENTER);
+ fdCacheTtl.left = new FormAttachment(middle, 0);
+ fdCacheTtl.right = new FormAttachment(100, 0);
+ wCacheTtl.setLayoutData(fdCacheTtl);
+ lastControl = wCacheTtl;
+
+ // --- Authentication type ---
+ Label wlAuthType = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlAuthType);
+ wlAuthType.setText(BaseMessages.getString(PKG, "S3Meta.AuthType.Label"));
+ FormData fdlAuthType = new FormData();
+ fdlAuthType.top = new FormAttachment(lastControl, 15);
+ fdlAuthType.left = new FormAttachment(0, 0);
+ fdlAuthType.right = new FormAttachment(middle, -margin);
+ wlAuthType.setLayoutData(fdlAuthType);
+ wAuthType = new Combo(parent, SWT.BORDER | SWT.SINGLE | SWT.LEFT | SWT.READ_ONLY);
+ PropsUi.setLook(wAuthType);
+ wAuthType.setItems(
+ new String[] {
+ BaseMessages.getString(PKG, "S3Meta.AuthType.Default"),
+ BaseMessages.getString(PKG, "S3Meta.AuthType.AccessKeys"),
+ BaseMessages.getString(PKG, "S3Meta.AuthType.CredentialsFile"),
+ });
+ FormData fdAuthType = new FormData();
+ fdAuthType.top = new FormAttachment(wlAuthType, 0, SWT.CENTER);
+ fdAuthType.left = new FormAttachment(middle, 0);
+ fdAuthType.right = new FormAttachment(100, 0);
+ wAuthType.setLayoutData(fdAuthType);
+ lastControl = wAuthType;
+
+ // --- Auth-specific composite ---
+ wAuthComposite = new Composite(parent, SWT.NONE);
+ PropsUi.setLook(wAuthComposite);
+ wAuthComposite.setLayout(new FormLayout());
+ FormData fdAuthComp = new FormData();
+ fdAuthComp.left = new FormAttachment(0, 0);
+ fdAuthComp.right = new FormAttachment(100, 0);
+ fdAuthComp.top = new FormAttachment(lastControl, margin);
+ fdAuthComp.bottom = new FormAttachment(100, 0);
+ wAuthComposite.setLayoutData(fdAuthComp);
+
+ buildAuthWidgets(wAuthComposite);
+
+ wAuthType.addListener(
+ SWT.Selection,
+ e -> {
+ setChanged();
+ updateAuthVisibility();
+ });
+
+ addModifyListeners();
+ setWidgetsContent();
+ resetChanged();
+ updateAuthVisibility();
+ }
+
+ private void buildAuthWidgets(Composite comp) {
+ // --- Keys block (Access key, Secret key, Session token) ---
+ wKeysComposite = new Composite(comp, SWT.NONE);
+ PropsUi.setLook(wKeysComposite);
+ wKeysComposite.setLayout(new FormLayout());
+ fdKeysComposite = new FormData();
+ fdKeysComposite.top = new FormAttachment(0, 0);
+ fdKeysComposite.left = new FormAttachment(0, 0);
+ fdKeysComposite.right = new FormAttachment(100, 0);
+ wKeysComposite.setLayoutData(fdKeysComposite);
+
+ wlAccessKey = new Label(wKeysComposite, SWT.RIGHT);
+ PropsUi.setLook(wlAccessKey);
+ wlAccessKey.setText(BaseMessages.getString(PKG, "S3Meta.AccessKey.Label"));
+ FormData fdl = new FormData();
+ fdl.top = new FormAttachment(0, INSIDE_PADDING);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlAccessKey.setLayoutData(fdl);
+ wAccessKey =
+ new TextVar(
+ manager.getVariables(),
+ wKeysComposite,
+ SWT.SINGLE | SWT.LEFT | SWT.BORDER | SWT.PASSWORD);
+ PropsUi.setLook(wAccessKey);
+ FormData fd = new FormData();
+ fd.top = new FormAttachment(wlAccessKey, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wAccessKey.setLayoutData(fd);
+
+ wlSecretKey = new Label(wKeysComposite, SWT.RIGHT);
+ PropsUi.setLook(wlSecretKey);
+ wlSecretKey.setText(BaseMessages.getString(PKG, "S3Meta.SecretKey.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(wAccessKey, margin);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlSecretKey.setLayoutData(fdl);
+ wSecretKey =
+ new TextVar(
+ manager.getVariables(),
+ wKeysComposite,
+ SWT.SINGLE | SWT.LEFT | SWT.BORDER | SWT.PASSWORD);
+ PropsUi.setLook(wSecretKey);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlSecretKey, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wSecretKey.setLayoutData(fd);
+
+ wlSessionToken = new Label(wKeysComposite, SWT.RIGHT);
+ PropsUi.setLook(wlSessionToken);
+ wlSessionToken.setText(BaseMessages.getString(PKG, "S3Meta.SessionToken.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(wSecretKey, margin);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlSessionToken.setLayoutData(fdl);
+ wSessionToken =
+ new TextVar(
+ manager.getVariables(),
+ wKeysComposite,
+ SWT.SINGLE | SWT.LEFT | SWT.BORDER | SWT.PASSWORD);
+ PropsUi.setLook(wSessionToken);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlSessionToken, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wSessionToken.setLayoutData(fd);
+
+ // --- Profile block (Credentials file, Profile name) ---
+ wProfileComposite = new Composite(comp, SWT.NONE);
+ PropsUi.setLook(wProfileComposite);
+ wProfileComposite.setLayout(new FormLayout());
+ fdProfileComposite = new FormData();
+ fdProfileComposite.top = new FormAttachment(0, 0);
+ fdProfileComposite.left = new FormAttachment(0, 0);
+ fdProfileComposite.right = new FormAttachment(100, 0);
+ wProfileComposite.setLayoutData(fdProfileComposite);
+
+ wlCredentialsFile = new Label(wProfileComposite, SWT.RIGHT);
+ PropsUi.setLook(wlCredentialsFile);
+ wlCredentialsFile.setText(BaseMessages.getString(PKG, "S3Meta.CredentialsFile.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(0, INSIDE_PADDING);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlCredentialsFile.setLayoutData(fdl);
+ wCredentialsFile =
+ new TextVar(manager.getVariables(), wProfileComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wCredentialsFile);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlCredentialsFile, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wCredentialsFile.setLayoutData(fd);
+
+ wlProfileName = new Label(wProfileComposite, SWT.RIGHT);
+ PropsUi.setLook(wlProfileName);
+ wlProfileName.setText(BaseMessages.getString(PKG, "S3Meta.ProfileName.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(wCredentialsFile, margin);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlProfileName.setLayoutData(fdl);
+ wProfileName =
+ new TextVar(manager.getVariables(), wProfileComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wProfileName);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlProfileName, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wProfileName.setLayoutData(fd);
+
+ // --- Common block (Endpoint, Path style) - position updated in updateAuthVisibility ---
+ wCommonComposite = new Composite(comp, SWT.NONE);
+ PropsUi.setLook(wCommonComposite);
+ wCommonComposite.setLayout(new FormLayout());
+ fdCommonComposite = new FormData();
+ fdCommonComposite.left = new FormAttachment(0, 0);
+ fdCommonComposite.right = new FormAttachment(100, 0);
+ wCommonComposite.setLayoutData(fdCommonComposite);
+
+ wlEndpoint = new Label(wCommonComposite, SWT.RIGHT);
+ PropsUi.setLook(wlEndpoint);
+ wlEndpoint.setText(BaseMessages.getString(PKG, "S3Meta.Endpoint.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(0, INSIDE_PADDING);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlEndpoint.setLayoutData(fdl);
+ wEndpoint =
+ new TextVar(manager.getVariables(), wCommonComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wEndpoint);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlEndpoint, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wEndpoint.setLayoutData(fd);
+
+ Label wlPathStyle = new Label(wCommonComposite, SWT.RIGHT);
+ PropsUi.setLook(wlPathStyle);
+ wlPathStyle.setText(BaseMessages.getString(PKG, "S3Meta.PathStyle.Label"));
+ fdl = new FormData();
+ fdl.top = new FormAttachment(wEndpoint, margin);
+ fdl.left = new FormAttachment(0, 0);
+ fdl.right = new FormAttachment(middle, -margin);
+ wlPathStyle.setLayoutData(fdl);
+ wPathStyle = new Button(wCommonComposite, SWT.CHECK | SWT.LEFT);
+ PropsUi.setLook(wPathStyle);
+ fd = new FormData();
+ fd.top = new FormAttachment(wlPathStyle, 0, SWT.CENTER);
+ fd.left = new FormAttachment(middle, 0);
+ fd.right = new FormAttachment(100, 0);
+ wPathStyle.setLayoutData(fd);
+ // Small spacer below Path-style (padding after last widget inside composite)
+ Label bottomSpacer = new Label(wCommonComposite, SWT.NONE);
+ FormData fdSpacer = new FormData();
+ fdSpacer.top = new FormAttachment(wPathStyle, margin);
+ fdSpacer.bottom = new FormAttachment(wPathStyle, margin + INSIDE_PADDING);
+ fdSpacer.left = new FormAttachment(0, 0);
+ fdSpacer.right = new FormAttachment(100, 0);
+ bottomSpacer.setLayoutData(fdSpacer);
+ }
+
+ private void updateAuthVisibility() {
+ int idx = wAuthType.getSelectionIndex();
+ if (idx < 0) {
+ idx = 0;
+ }
+ boolean keys = (idx == 1);
+ boolean profile = (idx == 2);
+ wKeysComposite.setVisible(keys);
+ wProfileComposite.setVisible(profile);
+ // Collapse hidden composites so they take no space (FormLayout reserves space for invisible
+ // controls otherwise)
+ if (keys) {
+ fdKeysComposite.top = new FormAttachment(0, 0);
+ fdKeysComposite.bottom = null;
+ } else {
+ fdKeysComposite.top = new FormAttachment(0, 0);
+ fdKeysComposite.bottom = new FormAttachment(0, 0);
+ }
+ if (profile) {
+ fdProfileComposite.top = new FormAttachment(0, 0);
+ fdProfileComposite.bottom = null;
+ } else {
+ fdProfileComposite.top = new FormAttachment(0, 0);
+ fdProfileComposite.bottom = new FormAttachment(0, 0);
+ }
+ wKeysComposite.setLayoutData(fdKeysComposite);
+ wProfileComposite.setLayoutData(fdProfileComposite);
+ if (keys) {
+ fdCommonComposite.top = new FormAttachment(wKeysComposite, margin);
+ } else if (profile) {
+ fdCommonComposite.top = new FormAttachment(wProfileComposite, margin);
+ } else {
+ fdCommonComposite.top = new FormAttachment(0, 0);
+ }
+ wCommonComposite.setLayoutData(fdCommonComposite);
+ wAuthComposite.layout(true, true);
+ }
+
+ private void addModifyListeners() {
+ wName.addListener(SWT.Modify, e -> setChanged());
+ wDescription.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wRegion.addListener(SWT.Modify, e -> setChanged());
+ wPartSize.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wCacheTtl.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wAccessKey.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wSecretKey.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wSessionToken.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wCredentialsFile.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wProfileName.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wEndpoint.getTextWidget().addListener(SWT.Modify, e -> setChanged());
+ wPathStyle.addListener(SWT.Selection, e -> setChanged());
+ }
+
+ @Override
+ public void setWidgetsContent() {
+ S3Meta meta = getMetadata();
+ wName.setText(Const.NVL(meta.getName(), ""));
+ wDescription.setText(Const.NVL(meta.getDescription(), ""));
+ String region = Const.NVL(meta.getRegion(), "");
+ wRegion.setText(region);
+ if (wRegion.indexOf(region) < 0 && StringUtils.isNotEmpty(region)) {
+ wRegion.add(region, 0);
+ wRegion.select(0);
+ } else if (wRegion.indexOf(region) >= 0) {
+ wRegion.select(wRegion.indexOf(region));
+ }
+ wPartSize.setText(Const.NVL(meta.getPartSize(), "5MB"));
+ wCacheTtl.setText(Const.NVL(meta.getCacheTtlSeconds(), "5"));
+
+ String authType = meta.getAuthenticationType();
+ if (StringUtils.isEmpty(authType)) {
+ authType = inferAuthType(meta);
+ }
+ authType = Const.NVL(authType, S3AuthType.DEFAULT.name());
+ int authIdx = 0;
+ if (S3AuthType.ACCESS_KEYS.name().equals(authType)) {
+ authIdx = 1;
+ } else if (S3AuthType.CREDENTIALS_FILE.name().equals(authType)) {
+ authIdx = 2;
+ }
+ wAuthType.select(authIdx);
+
+ wAccessKey.setText(Const.NVL(meta.getAccessKey(), ""));
+ wSecretKey.setText(Const.NVL(meta.getSecretKey(), ""));
+ wSessionToken.setText(Const.NVL(meta.getSessionToken(), ""));
+ wCredentialsFile.setText(Const.NVL(meta.getCredentialsFile(), ""));
+ wProfileName.setText(Const.NVL(meta.getProfileName(), ""));
+ wEndpoint.setText(Const.NVL(meta.getEndpoint(), ""));
+ wPathStyle.setSelection(meta.isPathStyleAccess());
+ updateAuthVisibility();
+ }
+
+ @Override
+ public void getWidgetsContent(S3Meta meta) {
+ meta.setName(wName.getText());
+ meta.setDescription(wDescription.getText());
+ meta.setRegion(StringUtils.isNotEmpty(wRegion.getText()) ? wRegion.getText().trim() : null);
+ meta.setPartSize(wPartSize.getText());
+ meta.setCacheTtlSeconds(wCacheTtl.getText());
+
+ int authIdx = wAuthType.getSelectionIndex();
+ if (authIdx == 1) {
+ meta.setAuthenticationType(S3AuthType.ACCESS_KEYS.name());
+ meta.setAccessKey(wAccessKey.getText());
+ meta.setSecretKey(wSecretKey.getText());
+ meta.setSessionToken(wSessionToken.getText());
+ meta.setCredentialsFile(null);
+ meta.setProfileName(null);
+ } else if (authIdx == 2) {
+ meta.setAuthenticationType(S3AuthType.CREDENTIALS_FILE.name());
+ meta.setAccessKey(null);
+ meta.setSecretKey(null);
+ meta.setSessionToken(null);
+ meta.setCredentialsFile(wCredentialsFile.getText());
+ meta.setProfileName(wProfileName.getText());
+ } else {
+ meta.setAuthenticationType(S3AuthType.DEFAULT.name());
+ meta.setAccessKey(null);
+ meta.setSecretKey(null);
+ meta.setSessionToken(null);
+ meta.setCredentialsFile(null);
+ meta.setProfileName(wProfileName.getText());
+ }
+ meta.setEndpoint(wEndpoint.getText());
+ meta.setPathStyleAccess(wPathStyle.getSelection());
+ }
+
+ @Override
+ public boolean setFocus() {
+ return wName != null && !wName.isDisposed() && wName.setFocus();
+ }
+
+ private static String inferAuthType(S3Meta meta) {
+ if (StringUtils.isNotEmpty(meta.getAccessKey())
+ || StringUtils.isNotEmpty(meta.getSecretKey())) {
+ return S3AuthType.ACCESS_KEYS.name();
+ }
+ if (StringUtils.isNotEmpty(meta.getCredentialsFile())) {
+ return S3AuthType.CREDENTIALS_FILE.name();
+ }
+ return S3AuthType.DEFAULT.name();
+ }
+
+ @Override
+ public void save() throws HopException {
+ super.save();
+ HopVfs.reset();
+ }
+}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileName.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileName.java
index 5f71d8c3a4..a5e50574e2 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileName.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileName.java
@@ -16,6 +16,7 @@
*/
package org.apache.hop.vfs.s3.s3.vfs;
+import lombok.Getter;
import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.provider.AbstractFileName;
@@ -24,8 +25,8 @@ import org.apache.commons.vfs2.provider.AbstractFileName;
public class S3FileName extends AbstractFileName {
public static final String DELIMITER = "/";
- private final String bucketId;
- private String bucketRelativePath;
+ @Getter private final String bucketId;
+ @Getter private String bucketRelativePath;
public S3FileName(String scheme, String bucketId, String path, FileType type) {
super(scheme, path, type);
@@ -50,14 +51,6 @@ public class S3FileName extends AbstractFileName {
return buffer.toString();
}
- public String getBucketId() {
- return bucketId;
- }
-
- public String getBucketRelativePath() {
- return bucketRelativePath;
- }
-
@Override
public FileName createName(String absPath, FileType type) {
return new S3FileName(getScheme(), bucketId, absPath, type);
@@ -66,17 +59,6 @@ public class S3FileName extends AbstractFileName {
@Override
protected void appendRootUri(StringBuilder buffer, boolean addPassword) {
buffer.append(getScheme());
- // Only 1 slash is needed here, because this class is not expecting an authority,
- // instead it is
- // expecting that the connection has already been established to the Amazon AWS S3 file system,
- // the second slash
- // comes from the absolute path of the file stored in the file system. So the root path with
- // this Uri ends up
- // being: s3:// instead of s3:///. A file located in a top level bucket would be
- // s3://bucket/example.txt instead of
- // s3:///bucket/example.txt. In our VFS, this is handled the same, in CLS 3 slashes do not
- // resolve appropriately.
- // For consistency, the code here changes so that we will end up with 2 slashes.
buffer.append("://");
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileObject.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileObject.java
index cbd28fcf5d..38f7354f93 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileObject.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileObject.java
@@ -17,23 +17,9 @@
package org.apache.hop.vfs.s3.s3.vfs;
-import static java.util.AbstractMap.SimpleEntry;
-
-import com.amazonaws.SdkClientException;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import java.io.InputStream;
import java.io.OutputStream;
-import java.util.List;
import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.provider.AbstractFileName;
-import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.vfs.s3.s3common.S3CommonFileObject;
import org.apache.hop.vfs.s3.s3common.S3CommonPipedOutputStream;
@@ -41,149 +27,16 @@ public class S3FileObject extends S3CommonFileObject {
public S3FileObject(final AbstractFileName name, final S3FileSystem fileSystem) {
super(name, fileSystem);
- // bucket name needs to be adjusted
- this.bucketName = getS3BucketName();
- }
-
- @Override
- public String getS3BucketName() {
- String s3BucketName = getName().getPath();
- if (s3BucketName.indexOf(DELIMITER, 1) > 1) {
- // this file is a file, to get the bucket, remove the name from the path
- s3BucketName = s3BucketName.substring(0, s3BucketName.indexOf(DELIMITER, 1));
- } else {
- // this file is a bucket
- s3BucketName = s3BucketName.replaceAll(DELIMITER, "");
- }
- if (s3BucketName.startsWith("/")) {
- s3BucketName = s3BucketName.substring(1);
- }
- return s3BucketName;
- }
-
- @Override
- protected List<String> getS3ObjectsFromVirtualFolder(String key, String bucket) {
- // see if bucket name needs to be adjusted from old driver pattern
- SimpleEntry<String, String> newPath = fixFilePath(key, bucket);
-
- return super.getS3ObjectsFromVirtualFolder(newPath.getKey(), newPath.getValue());
- }
-
- @Override
- protected S3Object getS3Object(String key, String bucket) {
- if (s3Object != null && s3Object.getObjectContent() != null) {
- LogChannel.GENERAL.logDebug("Returning existing object {0}", getQualifiedName());
- return s3Object;
- } else {
- LogChannel.GENERAL.logDebug("Getting object {0}", getQualifiedName());
- SimpleEntry<String, String> newPath = fixFilePath(key, bucket);
- return fileSystem.getS3Client().getObject(newPath.getValue(), newPath.getKey());
- }
- }
-
- private boolean bucketExists(String bucket) {
- boolean bucketExists = false;
- try {
- bucketExists = fileSystem.getS3Client().doesBucketExistV2(bucket);
- } catch (SdkClientException e) {
- LogChannel.GENERAL.logError("Exception checking if bucket exists", e);
- }
- return bucketExists;
- }
-
- @Override
- protected boolean isRootBucket() {
- SimpleEntry<String, String> newPath = fixFilePath(key, bucketName);
- return newPath.getKey().equals("");
}
@Override
public void doDelete() throws FileSystemException {
- // see if bucket name needs to be adjusted from old driver pattern
- SimpleEntry<String, String> newPath = fixFilePath(key, bucketName);
-
- doDelete(newPath.getKey(), newPath.getValue());
+ super.doDelete();
}
@Override
public OutputStream doGetOutputStream(boolean bAppend) throws Exception {
- SimpleEntry<String, String> newPath = fixFilePath(key, bucketName);
return new S3CommonPipedOutputStream(
- this.fileSystem,
- newPath.getValue(),
- newPath.getKey(),
- ((S3FileSystem) this.fileSystem).getPartSize());
- }
-
- @Override
- protected PutObjectRequest createPutObjectRequest(
- String bucketName, String key, InputStream inputStream, ObjectMetadata objectMetadata) {
- SimpleEntry<String, String> newPath = fixFilePath(key, bucketName);
- return new PutObjectRequest(newPath.getValue(), newPath.getKey(), inputStream, objectMetadata);
- }
-
- @Override
- protected CopyObjectRequest createCopyObjectRequest(
- String sourceBucket, String sourceKey, String destBucket, String destKey) {
- SimpleEntry<String, String> sourcePath = fixFilePath(sourceKey, sourceBucket);
- SimpleEntry<String, String> destPath = fixFilePath(destKey, destBucket);
- return new CopyObjectRequest(
- sourcePath.getValue(), sourcePath.getKey(), destPath.getValue(), destPath.getKey());
- }
-
- @Override
- public void handleAttachException(String key, String bucket) throws FileSystemException {
- SimpleEntry<String, String> newPath = fixFilePath(key, bucket);
- String keyWithDelimiter = newPath.getKey() + DELIMITER;
- try {
- s3Object = getS3Object(keyWithDelimiter, newPath.getValue());
- s3ObjectMetadata = s3Object.getObjectMetadata();
- injectType(FileType.FOLDER);
- } catch (AmazonS3Exception e2) {
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest()
- .withBucketName(newPath.getValue())
- .withPrefix(keyWithDelimiter)
- .withDelimiter(DELIMITER);
- ObjectListing ol = fileSystem.getS3Client().listObjects(listObjectsRequest);
-
- if (!(ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty())) {
- injectType(FileType.FOLDER);
- } else {
- // Folders don't really exist - they will generate a "NoSuchKey" exception
- String errorCode = e2.getErrorCode();
- // confirms key doesn't exist but connection okay
- if (!errorCode.equals("NoSuchKey")) {
- // bubbling up other connection errors
- LogChannel.GENERAL.logError(
- "Could not get information on " + getQualifiedName(),
- e2); // make sure this gets printed for the user
- throw new FileSystemException("vfs.provider/get-type.error", getQualifiedName(), e2);
- }
- }
- }
- }
-
- // See if the first name on the path is actually a bucket. If not, it's probably an old-style
- // path.
- public SimpleEntry<String, String> fixFilePath(String key, String bucket) {
- String newBucket = bucket;
- String newKey = key;
-
- // see if the folder exists; if not, it might be from an old path and the real bucket is in the
- // key
- if (!bucketExists(bucket)) {
- LogChannel.GENERAL.logDebug(
- "Bucket {} from original path not found, might be an old path from the old driver",
- bucket);
- if (key.split(DELIMITER).length > 1) {
- newBucket = key.split(DELIMITER)[0];
- newKey = key.replaceFirst(newBucket + DELIMITER, "");
- } else {
- newBucket = key;
- newKey = "";
- }
- }
- return new SimpleEntry<>(newKey, newBucket);
+ this.fileSystem, bucketName, key, ((S3FileSystem) this.fileSystem).getPartSize());
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileProvider.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileProvider.java
index 73ab583fd3..b35df18fa5 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileProvider.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3/vfs/S3FileProvider.java
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,10 +17,15 @@
package org.apache.hop.vfs.s3.s3.vfs;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileSystem;
import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.vfs.s3.metadata.S3AuthType;
+import org.apache.hop.vfs.s3.metadata.S3Meta;
import org.apache.hop.vfs.s3.s3common.S3CommonFileProvider;
+import org.apache.hop.vfs.s3.s3common.S3CommonFileSystemConfigBuilder;
public class S3FileProvider extends S3CommonFileProvider {
@@ -30,14 +35,88 @@ public class S3FileProvider extends S3CommonFileProvider {
/** User Information. */
public static final String ATTR_USER_INFO = "UI";
+ private final IVariables variables;
+ private final S3Meta s3Meta;
+
public S3FileProvider() {
super();
+ this.variables = null;
+ this.s3Meta = null;
+ setFileNameParser(S3FileNameParser.getInstance());
+ }
+
+ /** Named connection: use this provider for URLs like connectionName://bucket/key */
+ public S3FileProvider(IVariables variables, S3Meta s3Meta) {
+ super();
+ this.variables = variables;
+ this.s3Meta = s3Meta;
setFileNameParser(S3FileNameParser.getInstance());
}
@Override
public FileSystem doCreateFileSystem(
final FileName name, final FileSystemOptions fileSystemOptions) {
- return new S3FileSystem(name, fileSystemOptions);
+ FileSystemOptions options =
+ fileSystemOptions != null ? fileSystemOptions : getDefaultFileSystemOptions();
+
+ if (s3Meta != null && variables != null) {
+ S3CommonFileSystemConfigBuilder config = new S3CommonFileSystemConfigBuilder(options);
+ String authType = s3Meta.getAuthenticationType();
+ if (StringUtils.isEmpty(authType)) {
+ if (StringUtils.isNotEmpty(s3Meta.getAccessKey())
+ || StringUtils.isNotEmpty(s3Meta.getSecretKey())) {
+ authType = S3AuthType.ACCESS_KEYS.name();
+ } else if (StringUtils.isNotEmpty(s3Meta.getCredentialsFile())) {
+ authType = S3AuthType.CREDENTIALS_FILE.name();
+ } else {
+ authType = S3AuthType.DEFAULT.name();
+ }
+ }
+ String region = variables.resolve(s3Meta.getRegion());
+ String endpoint = variables.resolve(s3Meta.getEndpoint());
+
+ if (StringUtils.isNotEmpty(region)) {
+ config.setRegion(region);
+ }
+ if (StringUtils.isNotEmpty(endpoint)) {
+ config.setEndpoint(endpoint);
+ }
+ if (S3AuthType.ACCESS_KEYS.name().equals(authType)) {
+ String accessKey = variables.resolve(s3Meta.getAccessKey());
+ String secretKey = variables.resolve(s3Meta.getSecretKey());
+ String sessionToken = variables.resolve(s3Meta.getSessionToken());
+ if (StringUtils.isNotEmpty(accessKey)) {
+ config.setAccessKey(accessKey);
+ }
+ if (StringUtils.isNotEmpty(secretKey)) {
+ config.setSecretKey(secretKey);
+ }
+ if (StringUtils.isNotEmpty(sessionToken)) {
+ config.setSessionToken(sessionToken);
+ }
+ } else if (S3AuthType.CREDENTIALS_FILE.name().equals(authType)) {
+ String credentialsFile = variables.resolve(s3Meta.getCredentialsFile());
+ String profileName = variables.resolve(s3Meta.getProfileName());
+ if (StringUtils.isNotEmpty(credentialsFile)) {
+ config.setCredentialsFile(credentialsFile);
+ }
+ if (StringUtils.isNotEmpty(profileName)) {
+ config.setProfileName(profileName);
+ }
+ } else if (S3AuthType.DEFAULT.name().equals(authType)) {
+ String profileName = variables.resolve(s3Meta.getProfileName());
+ if (StringUtils.isNotEmpty(profileName)) {
+ config.setProfileName(profileName);
+ }
+ }
+ config.setPathStyleAccess(String.valueOf(s3Meta.isPathStyleAccess()));
+ String cacheTtl = variables.resolve(s3Meta.getCacheTtlSeconds());
+ if (StringUtils.isNotEmpty(cacheTtl)) {
+ config.setCacheTtlSeconds(cacheTtl);
+ }
+ options = config.getFileSystemOptions();
+ }
+
+ return new S3FileSystem(name, options);
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileName.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileName.java
deleted file mode 100644
index b96344a6e4..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileName.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.vfs.s3.s3a.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.provider.AbstractFileName;
-
-/** Custom filename that represents an S3A file with the bucket and its relative path */
-public class S3AFileName extends AbstractFileName {
- public static final String DELIMITER = "/";
-
- private final String bucketId;
- private String bucketRelativePath;
-
- public S3AFileName(String scheme, String bucketId, String path, FileType type) {
- super(scheme, path, type);
-
- this.bucketId = bucketId;
-
- if (path.length() > 1) {
- this.bucketRelativePath = path.substring(1);
- if (type.equals(FileType.FOLDER)) {
- this.bucketRelativePath += DELIMITER;
- }
- } else {
- this.bucketRelativePath = "";
- }
- }
-
- @Override
- public String getURI() {
- final StringBuilder buffer = new StringBuilder();
- appendRootUri(buffer, false);
- buffer.append(getPath());
- return buffer.toString();
- }
-
- public String getBucketId() {
- return bucketId;
- }
-
- public String getBucketRelativePath() {
- return bucketRelativePath;
- }
-
- @Override
- public FileName createName(String absPath, FileType type) {
- return new S3AFileName(getScheme(), bucketId, absPath, type);
- }
-
- @Override
- protected void appendRootUri(StringBuilder buffer, boolean addPassword) {
- buffer.append(getScheme());
- buffer.append("://");
- buffer.append(bucketId);
- }
-
- @Override
- public boolean equals(Object o) {
- return super.equals(o);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileNameParser.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileNameParser.java
deleted file mode 100644
index 367e151aae..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileNameParser.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3a.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.provider.AbstractFileNameParser;
-import org.apache.commons.vfs2.provider.FileNameParser;
-import org.apache.commons.vfs2.provider.UriParser;
-import org.apache.commons.vfs2.provider.VfsComponentContext;
-
-/** Custom parser for the s3a URL */
-public class S3AFileNameParser extends AbstractFileNameParser {
- private static final S3AFileNameParser INSTANCE = new S3AFileNameParser();
-
- private S3AFileNameParser() {
- super();
- }
-
- public static FileNameParser getInstance() {
- return INSTANCE;
- }
-
- @Override
- public FileName parseUri(VfsComponentContext context, FileName base, String uri)
- throws FileSystemException {
- StringBuilder name = new StringBuilder();
-
- String scheme = UriParser.extractScheme(uri, name);
- UriParser.canonicalizePath(name, 0, name.length(), this);
-
- // Normalize separators in the path
- UriParser.fixSeparators(name);
-
- // Normalise the path
- FileType fileType = UriParser.normalisePath(name);
-
- // Extract bucket name
- final String bucketName = UriParser.extractFirstElement(name);
-
- return new S3AFileName(scheme, bucketName, name.toString(), fileType);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileObject.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileObject.java
deleted file mode 100644
index a551306760..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileObject.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3a.vfs;
-
-import org.apache.commons.vfs2.provider.AbstractFileName;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileObject;
-
-public class S3AFileObject extends S3CommonFileObject {
-
- S3AFileObject(final AbstractFileName name, final S3AFileSystem fileSystem) {
- super(name, fileSystem);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileProvider.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileProvider.java
deleted file mode 100644
index 0a9400762e..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3a.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileSystem;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileProvider;
-
-public class S3AFileProvider extends S3CommonFileProvider {
-
- /** The scheme this provider was designed to support */
- public static final String SCHEME = "s3a";
-
- /** User Information. */
- public static final String ATTR_USER_INFO = "UI";
-
- public S3AFileProvider() {
- super();
- setFileNameParser(S3AFileNameParser.getInstance());
- }
-
- @Override
- protected FileSystem doCreateFileSystem(
- final FileName name, final FileSystemOptions fileSystemOptions) throws FileSystemException {
- return new S3AFileSystem(name, fileSystemOptions);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileSystem.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileSystem.java
deleted file mode 100644
index a4f4e46068..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3a/vfs/S3AFileSystem.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3a.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileObject;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.commons.vfs2.provider.AbstractFileName;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileSystem;
-
-public class S3AFileSystem extends S3CommonFileSystem {
-
- protected S3AFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions) {
- super(rootName, fileSystemOptions);
- }
-
- @Override
- protected FileObject createFile(AbstractFileName name) throws Exception {
- return new S3AFileObject(name, this);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileInputStream.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileInputStream.java
index b3ab75a696..8d72a52bdf 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileInputStream.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileInputStream.java
@@ -16,23 +16,26 @@
*/
package org.apache.hop.vfs.s3.s3common;
-import com.amazonaws.services.s3.model.S3Object;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.vfs2.util.MonitorInputStream;
+/** Wraps S3 getObject stream and closes the response when the stream is closed. */
public class S3CommonFileInputStream extends MonitorInputStream {
- private final S3Object s3Object;
+ private final Closeable toClose;
- public S3CommonFileInputStream(InputStream in, S3Object s3Object) {
+ public S3CommonFileInputStream(InputStream in, Closeable toClose) {
super(in);
- this.s3Object = s3Object;
+ this.toClose = toClose;
}
@Override
protected void onClose() throws IOException {
super.onClose();
- this.s3Object.close();
+ if (toClose != null) {
+ toClose.close();
+ }
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileObject.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileObject.java
index ecad08de57..dab53fb61f 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileObject.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileObject.java
@@ -17,38 +17,48 @@
package org.apache.hop.vfs.s3.s3common;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.Bucket;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.google.common.annotations.VisibleForTesting;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.time.Instant;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import lombok.Getter;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.provider.AbstractFileName;
import org.apache.commons.vfs2.provider.AbstractFileObject;
import org.apache.hop.core.logging.LogChannel;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.Bucket;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
public abstract class S3CommonFileObject extends AbstractFileObject {
public static final String DELIMITER = "/";
protected S3CommonFileSystem fileSystem;
- protected String bucketName;
- protected String key;
- protected S3Object s3Object;
- protected ObjectMetadata s3ObjectMetadata;
+ @Getter protected String bucketName;
+ @Getter protected String key;
+
+ protected ResponseInputStream<GetObjectResponse> currentObjectStream;
+ protected Long contentLength;
+ protected Instant lastModified;
protected S3CommonFileObject(final AbstractFileName name, final S3CommonFileSystem fileSystem) {
super(name, fileSystem);
@@ -59,15 +69,15 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
@Override
protected long doGetContentSize() {
- return s3ObjectMetadata.getContentLength();
+ return contentLength != null ? contentLength : 0;
}
@Override
protected InputStream doGetInputStream() throws Exception {
LogChannel.GENERAL.logDebug("Accessing content {0}", getQualifiedName());
closeS3Object();
- S3Object streamS3Object = getS3Object();
- return new S3CommonFileInputStream(streamS3Object.getObjectContent(), streamS3Object);
+ ResponseInputStream<GetObjectResponse> stream = getObjectStream(bucketName, key);
+ return new S3CommonFileInputStream(stream, stream);
}
@Override
@@ -78,45 +88,40 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
@Override
protected String[] doListChildren() throws Exception {
List<String> childrenList = new ArrayList<>();
-
- // only listing folders or the root bucket
if (getType() == FileType.FOLDER || isRootBucket()) {
childrenList = getS3ObjectsFromVirtualFolder(key, bucketName);
}
- String[] childrenArr = new String[childrenList.size()];
-
- return childrenList.toArray(childrenArr);
+ return childrenList.toArray(new String[0]);
}
protected String getS3BucketName() {
- String bucket = getName().getPath();
- if (bucket.indexOf(DELIMITER, 1) > 1) {
- // this file is a file, to get the bucket, remove the name from the path
- bucket = bucket.substring(1, bucket.indexOf(DELIMITER, 1));
- } else {
- // this file is a bucket
- bucket = bucket.replace(DELIMITER, "");
+ String path = getName().getPath();
+ if (path == null) {
+ return "";
}
- if (bucket.startsWith("/")) {
- bucket = bucket.substring(1);
+ if (path.startsWith(DELIMITER)) {
+ path = path.substring(1);
}
- return bucket;
+ if (path.isEmpty()) {
+ return "";
+ }
+ int slash = path.indexOf(DELIMITER);
+ return slash >= 0 ? path.substring(0, slash) : path;
}
protected List<String> getS3ObjectsFromVirtualFolder(String key, String bucketName) {
List<String> childrenList = new ArrayList<>();
-
- // fix cases where the path doesn't include the final delimiter
String realKey = key;
if (!realKey.endsWith(DELIMITER)) {
realKey += DELIMITER;
}
if ("".equals(key) && "".equals(bucketName)) {
- // Getting buckets in root folder
- List<Bucket> bucketList = fileSystem.getS3Client().listBuckets();
- for (Bucket bucket : bucketList) {
- childrenList.add(bucket.getName() + DELIMITER);
+ List<Bucket> buckets = fileSystem.getS3Client().listBuckets().buckets();
+ if (buckets != null) {
+ for (Bucket b : buckets) {
+ childrenList.add(b.name() + DELIMITER);
+ }
}
} else {
getObjectsFromNonRootFolder(key, bucketName, childrenList, realKey);
@@ -126,148 +131,227 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
private void getObjectsFromNonRootFolder(
String key, String bucketName, List<String> childrenList, String realKey) {
- // Getting files/folders in a folder/bucket
String prefix = key.isEmpty() || key.endsWith(DELIMITER) ? key : key + DELIMITER;
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest()
- .withBucketName(bucketName)
- .withPrefix(prefix)
- .withDelimiter(DELIMITER);
-
- ObjectListing ol = fileSystem.getS3Client().listObjects(listObjectsRequest);
-
- ArrayList<S3ObjectSummary> allSummaries = new ArrayList<>(ol.getObjectSummaries());
- ArrayList<String> allCommonPrefixes = new ArrayList<>(ol.getCommonPrefixes());
-
- // get full list
- while (ol.isTruncated()) {
- ol = fileSystem.getS3Client().listNextBatchOfObjects(ol);
- allSummaries.addAll(ol.getObjectSummaries());
- allCommonPrefixes.addAll(ol.getCommonPrefixes());
- }
-
- for (S3ObjectSummary s3os : allSummaries) {
- if (!s3os.getKey().equals(realKey)) {
- childrenList.add(s3os.getKey().substring(prefix.length()));
+ Map<String, S3ListCache.ChildInfo> cacheEntries = new LinkedHashMap<>();
+ String continuationToken = null;
+ do {
+ ListObjectsV2Request.Builder req =
+ ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).delimiter(DELIMITER);
+ if (continuationToken != null) {
+ req.continuationToken(continuationToken);
}
- }
-
- for (String commonPrefix : allCommonPrefixes) {
- if (!commonPrefix.equals(realKey)) {
- childrenList.add(commonPrefix.substring(prefix.length()));
+ ListObjectsV2Response response = fileSystem.getS3Client().listObjectsV2(req.build());
+
+ if (response.contents() != null) {
+ for (S3Object s3o : response.contents()) {
+ if (!s3o.key().equals(realKey)) {
+ String childName = s3o.key().substring(prefix.length());
+ childrenList.add(childName);
+ cacheEntries.put(
+ s3o.key(),
+ new S3ListCache.ChildInfo(
+ FileType.FILE,
+ s3o.size() != null ? s3o.size() : 0,
+ s3o.lastModified() != null ? s3o.lastModified() : Instant.EPOCH));
+ }
+ }
}
+ if (response.commonPrefixes() != null) {
+ for (CommonPrefix cp : response.commonPrefixes()) {
+ String p = cp.prefix();
+ if (!p.equals(realKey)) {
+ childrenList.add(p.substring(prefix.length()));
+ cacheEntries.put(p, new S3ListCache.ChildInfo(FileType.FOLDER, 0, Instant.EPOCH));
+ }
+ }
+ }
+ continuationToken = response.nextContinuationToken();
+ } while (continuationToken != null);
+
+ if (!cacheEntries.isEmpty()) {
+ fileSystem.putListCache(bucketName, prefix, cacheEntries);
}
}
protected String getBucketRelativeS3Path() {
- if (getName().getPath().indexOf(DELIMITER, 1) >= 0) {
- return getName().getPath().substring(getName().getPath().indexOf(DELIMITER, 1) + 1);
- } else {
+ String path = getName().getPath();
+ if (path == null) {
return "";
}
+ if (path.startsWith(DELIMITER)) {
+ path = path.substring(1);
+ }
+ if (path.isEmpty()) {
+ return "";
+ }
+ int slash = path.indexOf(DELIMITER);
+ return slash >= 0 ? path.substring(slash + 1) : "";
}
- @VisibleForTesting
- public S3Object getS3Object() {
- return getS3Object(this.key, this.bucketName);
- }
-
- protected S3Object getS3Object(String key, String bucket) {
- if (s3Object != null && s3Object.getObjectContent() != null) {
+ protected ResponseInputStream<GetObjectResponse> getObjectStream(String bucket, String key) {
+ if (currentObjectStream != null) {
LogChannel.GENERAL.logDebug("Returning existing object {0}", getQualifiedName());
- return s3Object;
- } else {
- LogChannel.GENERAL.logDebug("Getting object {0}", getQualifiedName());
- return fileSystem.getS3Client().getObject(bucket, key);
+ return currentObjectStream;
}
+ return fileSystem
+ .getS3Client()
+ .getObject(GetObjectRequest.builder().bucket(bucket).key(key).build());
}
protected boolean isRootBucket() {
return key.equals("");
}
+ private static String errorCode(S3Exception e) {
+ if (e.awsErrorDetails() != null && e.awsErrorDetails().errorCode() != null) {
+ return e.awsErrorDetails().errorCode();
+ }
+ return e.getMessage();
+ }
+
@Override
public void doAttach() throws Exception {
LogChannel.GENERAL.logDebug("Attach called on {0}", getQualifiedName());
injectType(FileType.IMAGINARY);
if (isRootBucket()) {
- // cannot attach to root bucket
injectType(FileType.FOLDER);
return;
}
+
+ // Use list cache when possible to avoid headObject or extra list calls
+ String parentPrefix = S3ListCache.parentPrefix(key);
+ S3ListCache.ChildInfo cached = fileSystem.getFromListCache(bucketName, parentPrefix, key);
+ if (cached == null && !key.endsWith(DELIMITER)) {
+ cached = fileSystem.getFromListCache(bucketName, parentPrefix, key + DELIMITER);
+ }
+ if (cached != null) {
+ contentLength = cached.size;
+ lastModified = cached.lastModified != null ? cached.lastModified : null;
+ injectType(cached.type);
+ if (cached.type == FileType.FOLDER) {
+ this.key = key.endsWith(DELIMITER) ? key : key + DELIMITER;
+ }
+ return;
+ }
+
try {
- // 1. Is it an existing file?
- s3ObjectMetadata = fileSystem.getS3Client().getObjectMetadata(bucketName, key);
- injectType(
- getName().getType()); // if this worked then the automatically detected type is right
- } catch (AmazonS3Exception e) { // S3 object doesn't exist
- // 2. Is it in reality a folder?
- handleAttachException(key, bucketName);
+ HeadObjectResponse head =
+ fileSystem
+ .getS3Client()
+ .headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build());
+ contentLength = head.contentLength();
+ lastModified = head.lastModified();
+ injectType(getName().getType());
+ } catch (S3Exception e) {
+ handleAttachException(key, bucketName, e);
} finally {
closeS3Object();
}
}
- protected void handleAttachException(String key, String bucket) throws IOException {
+ protected void handleAttachException(String key, String bucket, S3Exception e)
+ throws IOException {
String keyWithDelimiter = key + DELIMITER;
+ String err = errorCode(e);
+ boolean isNotFound = "404".equals(err) || "NotFound".equals(err) || "NoSuchKey".equals(err);
+
+ if (isNotFound) {
+ handleNotFoundAttach(bucket, keyWithDelimiter);
+ } else {
+ resolveTypeViaList(key, bucket, keyWithDelimiter);
+ }
+ }
+
+ private void handleNotFoundAttach(String bucket, String keyWithDelimiter) throws IOException {
try {
- s3ObjectMetadata = fileSystem.getS3Client().getObjectMetadata(bucketName, key);
- injectType(FileType.FOLDER);
- this.key = keyWithDelimiter;
- } catch (AmazonS3Exception e1) {
- String errorCode = e1.getErrorCode();
+ ResponseInputStream<GetObjectResponse> stream =
+ fileSystem
+ .getS3Client()
+ .getObject(GetObjectRequest.builder().bucket(bucket).key(keyWithDelimiter).build());
try {
- // S3 Object does not exist (could be the process of creating a new file. Lets fallback to
- // old the old behavior. (getting the s3 object)
- if (errorCode.equals("404 Not Found")) {
- s3Object = getS3Object(keyWithDelimiter, bucket);
- s3ObjectMetadata = s3Object.getObjectMetadata();
- injectType(FileType.FOLDER);
- this.key = keyWithDelimiter;
- } else {
- // The exception was not related with not finding the file
- handleAttachExceptionFallback(bucket, keyWithDelimiter, e1);
- }
- } catch (AmazonS3Exception e2) {
- // something went wrong getting the s3 object
- handleAttachExceptionFallback(bucket, keyWithDelimiter, e2);
+ GetObjectResponse r = stream.response();
+ contentLength = r.contentLength();
+ lastModified = r.lastModified();
+ injectType(FileType.FOLDER);
+ this.key = keyWithDelimiter;
+ } finally {
+ stream.close();
}
+ } catch (S3Exception e2) {
+ checkFolderViaList(bucket, keyWithDelimiter);
} finally {
closeS3Object();
}
}
- private void handleAttachExceptionFallback(
- String bucket, String keyWithDelimiter, AmazonS3Exception exception)
+ /**
+ * headObject failed with a non-404 error (e.g. 400 from cross-region). Use listObjectsV2 (which
+ * handles cross-region) to determine if the path is a file, folder, or imaginary.
+ */
+ private void resolveTypeViaList(String key, String bucket, String keyWithDelimiter)
throws FileSystemException {
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest()
- .withBucketName(bucket)
- .withPrefix(keyWithDelimiter)
- .withDelimiter(DELIMITER);
- ObjectListing ol = fileSystem.getS3Client().listObjects(listObjectsRequest);
-
- if (!(ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty())) {
- injectType(FileType.FOLDER);
- } else {
- // Folders don't really exist - they will generate a "NoSuchKey" exception
- // confirms key doesn't exist but connection okay
- String errorCode = exception.getErrorCode();
- if (!errorCode.equals("NoSuchKey")) {
- // bubbling up other connection errors
- LogChannel.GENERAL.logError(
- "Could not get information on " + getQualifiedName(),
- exception); // make sure this gets printed for the user
- throw new FileSystemException("vfs.provider/get-type.error", getQualifiedName(), exception);
+ try {
+ ListObjectsV2Response response =
+ fileSystem
+ .getS3Client()
+ .listObjectsV2(
+ ListObjectsV2Request.builder()
+ .bucket(bucket)
+ .prefix(key)
+ .delimiter(DELIMITER)
+ .maxKeys(2)
+ .build());
+
+ if (response.contents() != null) {
+ for (S3Object obj : response.contents()) {
+ if (obj.key().equals(key)) {
+ contentLength = obj.size();
+ lastModified = obj.lastModified();
+ injectType(getName().getType());
+ return;
+ }
+ }
}
+ if (response.commonPrefixes() != null) {
+ for (CommonPrefix cp : response.commonPrefixes()) {
+ if (cp.prefix().equals(keyWithDelimiter)) {
+ injectType(FileType.FOLDER);
+ this.key = keyWithDelimiter;
+ return;
+ }
+ }
+ }
+ } catch (S3Exception listError) {
+ LogChannel.GENERAL.logError("Could not get information on " + getQualifiedName(), listError);
+ throw new FileSystemException("vfs.provider/get-type.error", getQualifiedName(), listError);
+ }
+ }
+
+ private void checkFolderViaList(String bucket, String keyWithDelimiter)
+ throws FileSystemException {
+ ListObjectsV2Response response =
+ fileSystem
+ .getS3Client()
+ .listObjectsV2(
+ ListObjectsV2Request.builder()
+ .bucket(bucket)
+ .prefix(keyWithDelimiter)
+ .delimiter(DELIMITER)
+ .maxKeys(1)
+ .build());
+ boolean hasContent =
+ (response.contents() != null && !response.contents().isEmpty())
+ || (response.commonPrefixes() != null && !response.commonPrefixes().isEmpty());
+ if (hasContent) {
+ injectType(FileType.FOLDER);
}
}
private void closeS3Object() throws IOException {
- if (s3Object != null) {
- s3Object.close();
- s3Object = null;
+ if (currentObjectStream != null) {
+ currentObjectStream.close();
+ currentObjectStream = null;
}
}
@@ -283,25 +367,31 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
}
protected void doDelete(String key, String bucketName) throws FileSystemException {
- // can only delete folder if empty
if (getType() == FileType.FOLDER) {
-
- // list all children inside the folder
- ObjectListing ol = fileSystem.getS3Client().listObjects(bucketName, key);
- ArrayList<S3ObjectSummary> allSummaries = new ArrayList<>(ol.getObjectSummaries());
-
- // get full list
- while (ol.isTruncated()) {
- ol = fileSystem.getS3Client().listNextBatchOfObjects(ol);
- allSummaries.addAll(ol.getObjectSummaries());
- }
-
- for (S3ObjectSummary s3os : allSummaries) {
- fileSystem.getS3Client().deleteObject(bucketName, s3os.getKey());
- }
+ String prefix = key.isEmpty() || key.endsWith(DELIMITER) ? key : key + DELIMITER;
+ String continuationToken = null;
+ do {
+ ListObjectsV2Request.Builder req =
+ ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix);
+ if (continuationToken != null) {
+ req.continuationToken(continuationToken);
+ }
+ ListObjectsV2Response response = fileSystem.getS3Client().listObjectsV2(req.build());
+ if (response.contents() != null) {
+ for (S3Object s3o : response.contents()) {
+ fileSystem
+ .getS3Client()
+ .deleteObject(
+ DeleteObjectRequest.builder().bucket(bucketName).key(s3o.key()).build());
+ }
+ }
+ continuationToken = response.nextContinuationToken();
+ } while (continuationToken != null);
}
-
- fileSystem.getS3Client().deleteObject(bucketName, key);
+ fileSystem
+ .getS3Client()
+ .deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(key).build());
+ fileSystem.invalidateListCacheForParentOf(bucketName, key);
}
@Override
@@ -311,31 +401,25 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
@Override
public long doGetLastModifiedTime() {
- if (s3ObjectMetadata == null) {
- return 0;
- }
- return s3ObjectMetadata.getLastModified().getTime();
+ return lastModified != null ? lastModified.toEpochMilli() : 0;
}
@Override
protected void doCreateFolder() throws Exception {
if (!isRootBucket()) {
- // create meta-data for your folder and set content-length to 0
- ObjectMetadata metadata = new ObjectMetadata();
- metadata.setContentLength(0);
- metadata.setContentType("binary/octet-stream");
-
- // create empty content
- InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
-
- // create a PutObjectRequest passing the folder name suffixed by /
- PutObjectRequest putObjectRequest =
- createPutObjectRequest(bucketName, key + DELIMITER, emptyContent, metadata);
-
- // send request to S3 to create folder
try {
- fileSystem.getS3Client().putObject(putObjectRequest);
- } catch (AmazonS3Exception e) {
+ fileSystem
+ .getS3Client()
+ .putObject(
+ PutObjectRequest.builder()
+ .bucket(bucketName)
+ .key(key + DELIMITER)
+ .contentLength(0L)
+ .contentType("binary/octet-stream")
+ .build(),
+ RequestBody.fromBytes(new byte[0]));
+ fileSystem.invalidateListCacheForParentOf(bucketName, key);
+ } catch (S3Exception e) {
throw new FileSystemException("vfs.provider.local/create-folder.error", this, e);
}
} else {
@@ -343,46 +427,34 @@ public abstract class S3CommonFileObject extends AbstractFileObject {
}
}
- protected PutObjectRequest createPutObjectRequest(
- String bucketName, String key, InputStream inputStream, ObjectMetadata objectMetadata) {
- return new PutObjectRequest(bucketName, key, inputStream, objectMetadata);
- }
-
@Override
protected void doRename(FileObject newFile) throws Exception {
- // no folder renames on S3
if (getType().equals(FileType.FOLDER)) {
throw new FileSystemException("vfs.provider/rename-not-supported.error");
}
-
- s3ObjectMetadata = fileSystem.getS3Client().getObjectMetadata(bucketName, key);
-
- if (s3ObjectMetadata == null) {
- // object doesn't exist
+ HeadObjectResponse head =
+ fileSystem
+ .getS3Client()
+ .headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build());
+ if (head == null) {
throw new FileSystemException("vfs.provider/rename.error", this, newFile);
}
-
S3CommonFileObject dest = (S3CommonFileObject) newFile;
-
- // 1. copy the file
- CopyObjectRequest copyObjRequest =
- createCopyObjectRequest(bucketName, key, dest.bucketName, dest.key);
- fileSystem.getS3Client().copyObject(copyObjRequest);
-
- // 2. delete self
+ fileSystem
+ .getS3Client()
+ .copyObject(
+ CopyObjectRequest.builder()
+ .sourceBucket(bucketName)
+ .sourceKey(key)
+ .destinationBucket(dest.bucketName)
+ .destinationKey(dest.key)
+ .build());
+ fileSystem.invalidateListCacheForParentOf(bucketName, key);
+ fileSystem.invalidateListCacheForParentOf(dest.bucketName, dest.key);
delete();
}
- protected CopyObjectRequest createCopyObjectRequest(
- String sourceBucket, String sourceKey, String destBucket, String destKey) {
- return new CopyObjectRequest(sourceBucket, sourceKey, destBucket, destKey);
- }
-
protected String getQualifiedName() {
- return getQualifiedName(this);
- }
-
- protected String getQualifiedName(S3CommonFileObject s3nFileObject) {
- return s3nFileObject.bucketName + "/" + s3nFileObject.key;
+ return bucketName + "/" + key;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileProvider.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileProvider.java
index 4f0e8a6782..4ceb6aaacc 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileProvider.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileProvider.java
@@ -36,6 +36,12 @@ public abstract class S3CommonFileProvider extends AbstractOriginatingFileProvid
UserAuthenticationData.USERNAME, UserAuthenticationData.PASSWORD
};
+ private static final FileSystemOptions defaultOptions = new FileSystemOptions();
+
+ public static FileSystemOptions getDefaultFileSystemOptions() {
+ return defaultOptions;
+ }
+
/** The provider's capabilities. */
protected static final Collection<Capability> capabilities =
Collections.unmodifiableCollection(
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystem.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystem.java
index 12d1d732d5..0ee9944be0 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystem.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystem.java
@@ -17,19 +17,8 @@
package org.apache.hop.vfs.s3.s3common;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.auth.profile.ProfilesConfigFile;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import java.io.File;
+import java.net.URI;
+import java.nio.file.Paths;
import java.util.Collection;
import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileObject;
@@ -38,15 +27,41 @@ import org.apache.commons.vfs2.provider.AbstractFileName;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.vfs.s3.amazon.s3.S3Util;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
public abstract class S3CommonFileSystem extends AbstractFileSystem {
private String awsAccessKeyCache;
private String awsSecretKeyCache;
- private AmazonS3 client;
+ private S3Client client;
+
+ /** Cache for list results to avoid headObject/list calls when resolving child type. */
+ private final S3ListCache listCache;
protected S3CommonFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions) {
super(rootName, null, fileSystemOptions);
+ long ttlMs = S3ListCache.DEFAULT_TTL_MS;
+ if (fileSystemOptions != null) {
+ S3CommonFileSystemConfigBuilder cfg = new S3CommonFileSystemConfigBuilder(fileSystemOptions);
+ String cacheTtl = cfg.getCacheTtlSeconds();
+ if (!S3Util.isEmpty(cacheTtl)) {
+ try {
+ ttlMs = Long.parseLong(cacheTtl) * 1000L;
+ } catch (NumberFormatException e) {
+ LogChannel.GENERAL.logError("Invalid cacheTtlSeconds value: " + cacheTtl, e);
+ }
+ }
+ }
+ this.listCache = new S3ListCache(ttlMs);
}
@Override
@@ -57,70 +72,77 @@ public abstract class S3CommonFileSystem extends AbstractFileSystem {
@Override
protected abstract FileObject createFile(AbstractFileName name) throws Exception;
- public AmazonS3 getS3Client() {
+ public S3Client getS3Client() {
if (client == null && getFileSystemOptions() != null) {
- S3CommonFileSystemConfigBuilder s3CommonFileSystemConfigBuilder =
+ S3CommonFileSystemConfigBuilder cfg =
new S3CommonFileSystemConfigBuilder(getFileSystemOptions());
- String accessKey = s3CommonFileSystemConfigBuilder.getAccessKey();
- String secretKey = s3CommonFileSystemConfigBuilder.getSecretKey();
- String sessionToken = s3CommonFileSystemConfigBuilder.getSessionToken();
- String region = s3CommonFileSystemConfigBuilder.getRegion();
- String credentialsFilePath = s3CommonFileSystemConfigBuilder.getCredentialsFile();
- String profileName = s3CommonFileSystemConfigBuilder.getProfileName();
- String endpoint = s3CommonFileSystemConfigBuilder.getEndpoint();
- String signatureVersion = s3CommonFileSystemConfigBuilder.getSignatureVersion();
- String pathStyleAccess = s3CommonFileSystemConfigBuilder.getPathStyleAccess();
- boolean access = (pathStyleAccess == null) || Boolean.parseBoolean(pathStyleAccess);
- AWSCredentialsProvider awsCredentialsProvider = null;
- Regions regions = Regions.DEFAULT_REGION;
+ String accessKey = cfg.getAccessKey();
+ String secretKey = cfg.getSecretKey();
+ String sessionToken = cfg.getSessionToken();
+ String region = cfg.getRegion();
+ String credentialsFilePath = cfg.getCredentialsFile();
+ String profileName = cfg.getProfileName();
+ String endpoint = cfg.getEndpoint();
+ String pathStyleAccess = cfg.getPathStyleAccess();
+ boolean pathStyle = (pathStyleAccess == null) || Boolean.parseBoolean(pathStyleAccess);
+
+ AwsCredentialsProvider credentialsProvider = null;
+ Region awsRegion = Region.US_EAST_1;
+
if (!S3Util.isEmpty(accessKey) && !S3Util.isEmpty(secretKey)) {
- AWSCredentials awsCredentials;
if (S3Util.isEmpty(sessionToken)) {
- awsCredentials = new BasicAWSCredentials(accessKey, secretKey);
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey));
} else {
- awsCredentials = new BasicSessionCredentials(accessKey, secretKey, sessionToken);
+ credentialsProvider =
+ StaticCredentialsProvider.create(
+ AwsSessionCredentials.create(accessKey, secretKey, sessionToken));
}
- awsCredentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
- regions = S3Util.isEmpty(region) ? Regions.DEFAULT_REGION : Regions.fromName(region);
} else if (!S3Util.isEmpty(credentialsFilePath)) {
- ProfilesConfigFile profilesConfigFile = new ProfilesConfigFile(credentialsFilePath);
- awsCredentialsProvider = new ProfileCredentialsProvider(profilesConfigFile, profileName);
+ ProfileCredentialsProvider.Builder profileBuilder =
+ ProfileCredentialsProvider.builder()
+ .profileFile(
+ ProfileFile.builder()
+ .content(Paths.get(credentialsFilePath))
+ .type(ProfileFile.Type.CREDENTIALS)
+ .build())
+ .profileName(S3Util.isEmpty(profileName) ? "default" : profileName);
+ credentialsProvider = profileBuilder.build();
+ } else if (!S3Util.isEmpty(profileName)) {
+ credentialsProvider = ProfileCredentialsProvider.builder().profileName(profileName).build();
}
+ if (!S3Util.isEmpty(region)) {
+ awsRegion = Region.of(region);
+ }
+
+ boolean crossRegion = S3Util.isEmpty(region);
+ S3ClientBuilder builder =
+ S3Client.builder()
+ .crossRegionAccessEnabled(crossRegion)
+ .region(awsRegion)
+ .credentialsProvider(
+ credentialsProvider != null
+ ? credentialsProvider
+ : DefaultCredentialsProvider.create());
+
if (!S3Util.isEmpty(endpoint)) {
- ClientConfiguration clientConfiguration = new ClientConfiguration();
- clientConfiguration.setSignerOverride(
- S3Util.isEmpty(signatureVersion)
- ? S3Util.SIGNATURE_VERSION_SYSTEM_PROPERTY
- : signatureVersion);
- client =
- AmazonS3ClientBuilder.standard()
- .withEndpointConfiguration(
- new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName()))
- .withPathStyleAccessEnabled(access)
- .withClientConfiguration(clientConfiguration)
- .withCredentials(awsCredentialsProvider)
- .build();
- } else {
- client =
- AmazonS3ClientBuilder.standard()
- .enableForceGlobalBucketAccess()
- .withRegion(regions)
- .withCredentials(awsCredentialsProvider)
- .build();
+ builder.endpointOverride(URI.create(endpoint)).forcePathStyle(pathStyle);
}
+
+ client = builder.build();
}
if (client == null || hasClientChangedCredentials()) {
try {
- if (isRegionSet()) {
- client = AmazonS3ClientBuilder.standard().enableForceGlobalBucketAccess().build();
- } else {
- client =
- AmazonS3ClientBuilder.standard()
- .enableForceGlobalBucketAccess()
- .withRegion(Regions.DEFAULT_REGION)
- .build();
- }
+ String regionStr = System.getenv(S3Util.AWS_REGION);
+ Region region = regionStr != null ? Region.of(regionStr) : Region.US_EAST_1;
+ boolean crossRegion = S3Util.isEmpty(regionStr);
+ client =
+ S3Client.builder()
+ .crossRegionAccessEnabled(crossRegion)
+ .region(region)
+ .credentialsProvider(DefaultCredentialsProvider.create())
+ .build();
awsAccessKeyCache = System.getProperty(S3Util.ACCESS_KEY_SYSTEM_PROPERTY);
awsSecretKeyCache = System.getProperty(S3Util.SECRET_KEY_SYSTEM_PROPERTY);
} catch (Throwable t) {
@@ -138,19 +160,25 @@ public abstract class S3CommonFileSystem extends AbstractFileSystem {
awsSecretKeyCache, System.getProperty(S3Util.SECRET_KEY_SYSTEM_PROPERTY)));
}
- private boolean isRegionSet() {
- // region is set if explicitly set in env variable or configuration file is explicitly set
- if (System.getenv(S3Util.AWS_REGION) != null || System.getenv(S3Util.AWS_CONFIG_FILE) != null) {
- return true;
- }
- // check if configuration file exists in default location
- File awsConfigFolder =
- new File(
- System.getProperty("user.home")
- + File.separator
- + S3Util.AWS_FOLDER
- + File.separator
- + S3Util.CONFIG_FILE);
- return awsConfigFolder.exists();
+ /** Put list result into cache so child getType()/exists() can avoid network calls. */
+ public void putListCache(
+ String bucket, String prefix, java.util.Map<String, S3ListCache.ChildInfo> childEntries) {
+ listCache.put(bucket, prefix, childEntries);
+ }
+
+ /** Get cached type/size/lastModified for a child, or null if not cached or expired. */
+ public S3ListCache.ChildInfo getFromListCache(
+ String bucket, String parentPrefix, String childFullKey) {
+ return listCache.get(bucket, parentPrefix, childFullKey);
+ }
+
+ /** Invalidate list cache for the given prefix (e.g. after put/delete). */
+ public void invalidateListCache(String bucket, String prefix) {
+ listCache.invalidate(bucket, prefix);
+ }
+
+ /** Invalidate list cache for the parent directory of the given key. */
+ public void invalidateListCacheForParentOf(String bucket, String key) {
+ listCache.invalidateParentOf(bucket, key);
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystemConfigBuilder.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystemConfigBuilder.java
index e7279791f4..ddaa6dfc6f 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystemConfigBuilder.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonFileSystemConfigBuilder.java
@@ -17,10 +17,12 @@
package org.apache.hop.vfs.s3.s3common;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.commons.vfs2.FileSystem;
import org.apache.commons.vfs2.FileSystemConfigBuilder;
import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileSystem;
+import org.apache.hop.vfs.s3.s3.vfs.S3FileSystem;
/** Configuration Builder for S3 File System */
public class S3CommonFileSystemConfigBuilder extends FileSystemConfigBuilder {
@@ -32,23 +34,15 @@ public class S3CommonFileSystemConfigBuilder extends FileSystemConfigBuilder {
private static final String CREDENTIALS_FILE = "credentialsFile";
private static final String PROFILE_NAME = "profileName";
private static final String ENDPOINT = "endpoint";
- private static final String SIGNATURE_VERSION = "signature_version";
private static final String PATH_STYLE_ACCESS = "pathSyleAccess";
+ private static final String CACHE_TTL_SECONDS = "cacheTtlSeconds";
- private FileSystemOptions fileSystemOptions;
+ @Getter @Setter private FileSystemOptions fileSystemOptions;
public S3CommonFileSystemConfigBuilder(FileSystemOptions fileSystemOptions) {
this.fileSystemOptions = fileSystemOptions;
}
- public FileSystemOptions getFileSystemOptions() {
- return fileSystemOptions;
- }
-
- public void setFileSystemOptions(FileSystemOptions fileSystemOptions) {
- this.fileSystemOptions = fileSystemOptions;
- }
-
public void setAccessKey(String accessKey) {
this.setParam(getFileSystemOptions(), ACCESS_KEY, accessKey);
}
@@ -105,14 +99,6 @@ public class S3CommonFileSystemConfigBuilder extends FileSystemConfigBuilder {
return (String) this.getParam(getFileSystemOptions(), ENDPOINT);
}
- public void setSignatureVersion(String signatureVersion) {
- this.setParam(getFileSystemOptions(), SIGNATURE_VERSION, signatureVersion);
- }
-
- public String getSignatureVersion() {
- return (String) this.getParam(getFileSystemOptions(), SIGNATURE_VERSION);
- }
-
public void setPathStyleAccess(String pathStyleAccess) {
this.setParam(getFileSystemOptions(), PATH_STYLE_ACCESS, pathStyleAccess);
}
@@ -121,8 +107,16 @@ public class S3CommonFileSystemConfigBuilder extends FileSystemConfigBuilder {
return (String) this.getParam(getFileSystemOptions(), PATH_STYLE_ACCESS);
}
+ public void setCacheTtlSeconds(String cacheTtlSeconds) {
+ this.setParam(getFileSystemOptions(), CACHE_TTL_SECONDS, cacheTtlSeconds);
+ }
+
+ public String getCacheTtlSeconds() {
+ return (String) this.getParam(getFileSystemOptions(), CACHE_TTL_SECONDS);
+ }
+
@Override
protected Class<? extends FileSystem> getConfigClass() {
- return S3NFileSystem.class;
+ return S3FileSystem.class;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonPipedOutputStream.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonPipedOutputStream.java
index 29ae3fab1c..db1f66b969 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonPipedOutputStream.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonPipedOutputStream.java
@@ -17,12 +17,6 @@
package org.apache.hop.vfs.s3.s3common;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.UploadPartRequest;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -34,29 +28,38 @@ import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.util.StorageUnitConverter;
import org.apache.hop.i18n.BaseMessages;
-
-/** Custom OutputStream that enables chunked uploads into S3 */
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+
+/** Custom OutputStream that enables chunked uploads into S3 using AWS SDK v2. */
public class S3CommonPipedOutputStream extends PipedOutputStream {
private static final Class<?> PKG = S3CommonPipedOutputStream.class;
- /** set to aws multipart minimum 5MB. */
private static final int DEFAULT_PART_SIZE = 5 * 1024 * 1024;
private final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
private boolean initialized = false;
- private boolean blockedUntilDone = true;
+ @Getter @Setter private boolean blockedUntilDone = true;
private final PipedInputStream pipedInputStream;
private final S3AsyncTransferRunner s3AsyncTransferRunner;
private final S3CommonFileSystem fileSystem;
private Future<Boolean> result = null;
private final String bucketId;
private final String key;
-
- /** AWS Multipart part size. */
private final int partSize;
public S3CommonPipedOutputStream(S3CommonFileSystem fileSystem, String bucketId, String key)
@@ -67,14 +70,11 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
public S3CommonPipedOutputStream(
S3CommonFileSystem fileSystem, String bucketId, String key, int partSize) throws IOException {
this.pipedInputStream = new PipedInputStream();
-
try {
this.pipedInputStream.connect(this);
} catch (IOException e) {
- // FATAL, unexpected
throw new IOException("could not connect to pipedInputStream", e);
}
-
this.s3AsyncTransferRunner = new S3AsyncTransferRunner();
this.bucketId = bucketId;
this.key = key;
@@ -85,18 +85,10 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
private void initializeWrite() {
if (!initialized) {
initialized = true;
- result = this.executor.submit(s3AsyncTransferRunner);
+ result = executor.submit(s3AsyncTransferRunner);
}
}
- public boolean isBlockedUntilDone() {
- return blockedUntilDone;
- }
-
- public void setBlockedUntilDone(boolean blockedUntilDone) {
- this.blockedUntilDone = blockedUntilDone;
- }
-
@Override
public void write(int b) throws IOException {
initializeWrite();
@@ -112,7 +104,6 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
@Override
public void close() throws IOException {
super.close();
-
if (initialized && isBlockedUntilDone()) {
while (!result.isDone()) {
try {
@@ -124,7 +115,7 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
}
}
}
- this.executor.shutdown();
+ executor.shutdown();
}
class S3AsyncTransferRunner implements Callable<Boolean> {
@@ -132,85 +123,79 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
@Override
public Boolean call() throws Exception {
boolean returnVal = true;
- List<PartETag> partETags = new ArrayList<>();
-
- // Transform 1: Initialize
- InitiateMultipartUploadRequest initRequest;
- initRequest = new InitiateMultipartUploadRequest(bucketId, key);
-
- InitiateMultipartUploadResult initResponse = null;
+ List<CompletedPart> completedParts = new ArrayList<>();
+ String uploadId = null;
+ S3Client s3 = fileSystem.getS3Client();
- // NOTE: byte[] max size is ~2GB < 5GB = aws api max part size
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(partSize);
BufferedInputStream bis = new BufferedInputStream(pipedInputStream, partSize)) {
- initResponse = fileSystem.getS3Client().initiateMultipartUpload(initRequest);
- // Transform 2: Upload parts.
+
+ CreateMultipartUploadResponse createResponse =
+ s3.createMultipartUpload(
+ CreateMultipartUploadRequest.builder().bucket(bucketId).key(key).build());
+ uploadId = createResponse.uploadId();
+
byte[] tmpBuffer = new byte[partSize];
- int read = 0;
+ int read;
long offset = 0;
long totalRead = 0;
int partNum = 1;
- S3CommonWindowedSubstream s3is;
LogChannel.GENERAL.logDetailed(BaseMessages.getString(PKG, "INFO.S3MultiPart.Start"));
while ((read = bis.read(tmpBuffer)) >= 0) {
-
- // if something was actually read
if (read > 0) {
baos.write(tmpBuffer, 0, read);
totalRead += read;
}
- if (totalRead > partSize) {
- s3is = new S3CommonWindowedSubstream(baos.toByteArray());
-
- UploadPartRequest uploadRequest =
- new UploadPartRequest()
- .withBucketName(bucketId)
- .withKey(key)
- .withUploadId(initResponse.getUploadId())
- .withPartNumber(partNum++)
- .withFileOffset(offset)
- .withPartSize(totalRead)
- .withInputStream(s3is);
-
- // Upload part and add response to our list.
+ if (totalRead >= partSize) {
+ byte[] partBytes = baos.toByteArray();
+ UploadPartResponse uploadResp =
+ s3.uploadPart(
+ UploadPartRequest.builder()
+ .bucket(bucketId)
+ .key(key)
+ .uploadId(uploadId)
+ .partNumber(partNum++)
+ .build(),
+ RequestBody.fromBytes(partBytes));
+ completedParts.add(
+ CompletedPart.builder().eTag(uploadResp.eTag()).partNumber(partNum - 1).build());
LogChannel.GENERAL.logDetailed(
BaseMessages.getString(
PKG, "INFO.S3MultiPart.Upload", partNum - 1, offset, Long.toString(totalRead)));
- partETags.add(fileSystem.getS3Client().uploadPart(uploadRequest).getPartETag());
-
offset += totalRead;
- totalRead = 0; // reset part size counter
- baos.reset(); // reset output stream to 0
+ totalRead = 0;
+ baos.reset();
}
}
- // Transform 2.1 upload last part
- s3is = new S3CommonWindowedSubstream(baos.toByteArray());
-
- UploadPartRequest uploadRequest =
- new UploadPartRequest()
- .withBucketName(bucketId)
- .withKey(key)
- .withUploadId(initResponse.getUploadId())
- .withPartNumber(partNum++)
- .withFileOffset(offset)
- .withPartSize(totalRead)
- .withInputStream(s3is)
- .withLastPart(true);
-
- LogChannel.GENERAL.logDetailed(
- BaseMessages.getString(PKG, "INFO.S3MultiPart.Upload", partNum - 1, offset, totalRead));
- partETags.add(fileSystem.getS3Client().uploadPart(uploadRequest).getPartETag());
-
- // Transform 3: Complete.
- LogChannel.GENERAL.logDetailed(BaseMessages.getString(PKG, "INFO.S3MultiPart.Complete"));
- CompleteMultipartUploadRequest compRequest =
- new CompleteMultipartUploadRequest(
- bucketId, key, initResponse.getUploadId(), partETags);
+ if (totalRead > 0 || partNum == 1) {
+ byte[] lastPart = baos.toByteArray();
+ UploadPartResponse uploadResp =
+ s3.uploadPart(
+ UploadPartRequest.builder()
+ .bucket(bucketId)
+ .key(key)
+ .uploadId(uploadId)
+ .partNumber(partNum)
+ .build(),
+ RequestBody.fromBytes(lastPart));
+ completedParts.add(
+ CompletedPart.builder().eTag(uploadResp.eTag()).partNumber(partNum).build());
+ LogChannel.GENERAL.logDetailed(
+ BaseMessages.getString(PKG, "INFO.S3MultiPart.Upload", partNum, offset, totalRead));
+ }
- fileSystem.getS3Client().completeMultipartUpload(compRequest);
+ LogChannel.GENERAL.logDetailed(BaseMessages.getString(PKG, "INFO.S3MultiPart.Complete"));
+ s3.completeMultipartUpload(
+ CompleteMultipartUploadRequest.builder()
+ .bucket(bucketId)
+ .key(key)
+ .uploadId(uploadId)
+ .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
+ .build());
+ fileSystem.invalidateListCacheForParentOf(bucketId, key);
} catch (OutOfMemoryError oome) {
LogChannel.GENERAL.logError(
BaseMessages.getString(
@@ -222,18 +207,21 @@ public class S3CommonPipedOutputStream extends PipedOutputStream {
} catch (Exception e) {
LogChannel.GENERAL.logError(
BaseMessages.getString(PKG, "ERROR.S3MultiPart.ExceptionCaught"), e);
- if (initResponse == null) {
- close();
- } else {
- fileSystem
- .getS3Client()
- .abortMultipartUpload(
- new AbortMultipartUploadRequest(bucketId, key, initResponse.getUploadId()));
+ if (uploadId != null) {
+ try {
+ s3.abortMultipartUpload(
+ AbortMultipartUploadRequest.builder()
+ .bucket(bucketId)
+ .key(key)
+ .uploadId(uploadId)
+ .build());
+ } catch (Exception ignore) {
+ // ignore
+ }
LogChannel.GENERAL.logError(BaseMessages.getString(PKG, "ERROR.S3MultiPart.Aborted"));
}
returnVal = false;
}
-
return returnVal;
}
}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonWindowedSubstream.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonWindowedSubstream.java
deleted file mode 100644
index a18b9a4c7b..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3CommonWindowedSubstream.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3common;
-
-import java.io.ByteArrayInputStream;
-
-/** Custom InputStream that only virtually skips data to mimic a long running stream */
-public class S3CommonWindowedSubstream extends ByteArrayInputStream {
-
- public S3CommonWindowedSubstream(byte[] buf) {
- super(buf);
- }
-
- @Override
- public synchronized long skip(long n) {
- // virtual skip
- return n;
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3ListCache.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3ListCache.java
new file mode 100644
index 0000000000..15df491f02
--- /dev/null
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3common/S3ListCache.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.s3.s3common;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.vfs2.FileType;
+
+/**
+ * Cache for S3 list results. When a directory is listed via listObjectsV2, we store the child keys
+ * and their types/sizes so that later getType()/exists() for those children can be answered without
+ * headObject or extra list calls.
+ */
+public class S3ListCache {
+
+ /** Default TTL in milliseconds (10 seconds) — used when no metadata object is available. */
+ public static final long DEFAULT_TTL_MS = 10_000L;
+
+ /** Metadata for one child entry from a list response. */
+ public static final class ChildInfo {
+ public final FileType type;
+ public final long size;
+ public final Instant lastModified;
+
+ public ChildInfo(FileType type, long size, Instant lastModified) {
+ this.type = type;
+ this.size = size;
+ this.lastModified = lastModified;
+ }
+ }
+
+ private final long ttlMs;
+ private final ConcurrentHashMap<String, CachedList> cache = new ConcurrentHashMap<>();
+
+ public S3ListCache() {
+ this(DEFAULT_TTL_MS);
+ }
+
+ public S3ListCache(long ttlMs) {
+ this.ttlMs = ttlMs;
+ }
+
+ private static String cacheKey(String bucket, String prefix) {
+ return bucket + "|" + prefix;
+ }
+
+ private static class CachedList {
+ final Map<String, ChildInfo> entries;
+ final long expiryMillis;
+
+ CachedList(Map<String, ChildInfo> entries, long expiryMillis) {
+ this.entries = new ConcurrentHashMap<>(entries);
+ this.expiryMillis = expiryMillis;
+ }
+ }
+
+ /** Store list result for the given bucket and prefix. */
+ public void put(String bucket, String prefix, Map<String, ChildInfo> childEntries) {
+ if (childEntries == null || childEntries.isEmpty()) {
+ return;
+ }
+ String key = cacheKey(bucket, prefix);
+ cache.put(key, new CachedList(childEntries, System.currentTimeMillis() + ttlMs));
+ }
+
+ /**
+ * Look up cached type/size/lastModified for a child. Parent prefix is the directory path (e.g.
+ * "path/to/"). Child full key is the full S3 key (e.g. "path/to/file" or "path/to/subdir/").
+ * Returns null if not in cache or cache expired.
+ */
+ public ChildInfo get(String bucket, String parentPrefix, String childFullKey) {
+ String key = cacheKey(bucket, parentPrefix);
+ CachedList cached = cache.get(key);
+ if (cached == null) {
+ return null;
+ }
+ if (System.currentTimeMillis() > cached.expiryMillis) {
+ cache.remove(key);
+ return null;
+ }
+ return cached.entries.get(childFullKey);
+ }
+
+ /** Invalidate the list cache for the given bucket and prefix. */
+ public void invalidate(String bucket, String prefix) {
+ cache.remove(cacheKey(bucket, prefix));
+ }
+
+ /**
+ * Invalidate the list cache for the parent directory of the given key. Call after put/delete so
+ * the next list reflects the change.
+ */
+ public void invalidateParentOf(String bucket, String key) {
+ String parentPrefix = parentPrefix(key);
+ invalidate(bucket, parentPrefix);
+ }
+
+ /** Compute parent prefix: "path/to/" for key "path/to/file", "" for "file" or "". */
+ public static String parentPrefix(String key) {
+ if (key == null || key.isEmpty()) {
+ return "";
+ }
+ int last = key.lastIndexOf('/');
+ return last >= 0 ? key.substring(0, last + 1) : "";
+ }
+}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileName.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileName.java
deleted file mode 100644
index 87020875f1..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileName.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3n.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.provider.AbstractFileName;
-
-/** Custom filename that represents an S3 file with the bucket and its relative path */
-public class S3NFileName extends AbstractFileName {
- public static final String DELIMITER = "/";
-
- private String bucketId;
- private String bucketRelativePath;
-
- public S3NFileName(String scheme, String bucketId, String path, FileType type) {
- super(scheme, path, type);
-
- this.bucketId = bucketId;
-
- if (path.length() > 1) {
- this.bucketRelativePath = path.substring(1);
- if (type.equals(FileType.FOLDER)) {
- this.bucketRelativePath += DELIMITER;
- }
- } else {
- this.bucketRelativePath = "";
- }
- }
-
- @Override
- public String getURI() {
- final StringBuilder buffer = new StringBuilder();
- appendRootUri(buffer, false);
- buffer.append(getPath());
- return buffer.toString();
- }
-
- public String getBucketId() {
- return bucketId;
- }
-
- public String getBucketRelativePath() {
- return bucketRelativePath;
- }
-
- @Override
- public FileName createName(String absPath, FileType type) {
- return new S3NFileName(getScheme(), bucketId, absPath, type);
- }
-
- @Override
- protected void appendRootUri(StringBuilder buffer, boolean addPassword) {
- buffer.append(getScheme());
- buffer.append("://");
- buffer.append(bucketId);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileNameParser.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileNameParser.java
deleted file mode 100644
index e198eb9895..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileNameParser.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3n.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.provider.AbstractFileNameParser;
-import org.apache.commons.vfs2.provider.FileNameParser;
-import org.apache.commons.vfs2.provider.UriParser;
-import org.apache.commons.vfs2.provider.VfsComponentContext;
-
-/** Custom parser for the s3 URL */
-public class S3NFileNameParser extends AbstractFileNameParser {
- private static final S3NFileNameParser INSTANCE = new S3NFileNameParser();
-
- public S3NFileNameParser() {
- super();
- }
-
- public static FileNameParser getInstance() {
- return INSTANCE;
- }
-
- @Override
- public FileName parseUri(VfsComponentContext context, FileName base, String uri)
- throws FileSystemException {
- StringBuilder name = new StringBuilder();
-
- String scheme = UriParser.extractScheme(uri, name);
- UriParser.canonicalizePath(name, 0, name.length(), this);
-
- // Normalize separators in the path
- UriParser.fixSeparators(name);
-
- // Normalise the path
- FileType fileType = UriParser.normalisePath(name);
-
- // Extract bucket name
- final String bucketName = UriParser.extractFirstElement(name);
-
- return new S3NFileName(scheme, bucketName, name.toString(), fileType);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileProvider.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileProvider.java
deleted file mode 100644
index 020109816f..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3n.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileSystem;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileProvider;
-
-public class S3NFileProvider extends S3CommonFileProvider {
-
- /** The scheme this provider was designed to support */
- public static final String SCHEME = "s3n";
-
- /** User Information. */
- public static final String ATTR_USER_INFO = "UI";
-
- public S3NFileProvider() {
- super();
- setFileNameParser(S3NFileNameParser.getInstance());
- }
-
- @Override
- public FileSystem doCreateFileSystem(
- final FileName name, final FileSystemOptions fileSystemOptions) throws FileSystemException {
- return new S3NFileSystem(name, fileSystemOptions);
- }
-}
diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileSystem.java b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileSystem.java
deleted file mode 100644
index 3434774cd9..0000000000
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/s3n/vfs/S3NFileSystem.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.s3n.vfs;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileObject;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.commons.vfs2.provider.AbstractFileName;
-import org.apache.hop.vfs.s3.s3common.S3CommonFileSystem;
-
-public class S3NFileSystem extends S3CommonFileSystem {
-
- public S3NFileSystem(final FileName rootName, final FileSystemOptions fileSystemOptions) {
- super(rootName, fileSystemOptions);
- }
-
- @Override
- public FileObject createFile(AbstractFileName name) throws Exception {
- return new S3NFileObject(name, this);
- }
-}
diff --git a/plugins/tech/aws/src/main/resources/org/apache/hop/vfs/s3/metadata/messages/messages_en_US.properties b/plugins/tech/aws/src/main/resources/org/apache/hop/vfs/s3/metadata/messages/messages_en_US.properties
new file mode 100644
index 0000000000..80328ea054
--- /dev/null
+++ b/plugins/tech/aws/src/main/resources/org/apache/hop/vfs/s3/metadata/messages/messages_en_US.properties
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+S3Meta.name = S3 Connection
+S3Meta.description = Named S3 connection for VFS (AWS or S3-compatible)
+S3Meta.Name = S3 Connection
+S3Meta.Description = Named S3 connection for VFS (AWS or S3-compatible)
+S3Meta.Name.Label = Name
+S3Meta.Description.Label = Description
+S3Meta.Description.Description = Optional description of this connection
+S3Meta.AccessKey.Label = Access key
+S3Meta.AccessKey.Description = AWS access key ID (leave empty to use profile or default chain)
+S3Meta.SecretKey.Label = Secret key
+S3Meta.SecretKey.Description = AWS secret access key
+S3Meta.SessionToken.Label = Session token
+S3Meta.SessionToken.Description = Optional session token for temporary credentials
+S3Meta.Region.Label = Region
+S3Meta.Region.Description = AWS region (e.g. us-east-1)
+S3Meta.Endpoint.Label = Endpoint URL
+S3Meta.Endpoint.Description = Custom endpoint (e.g. for S3-compatible or MinIO)
+S3Meta.PathStyle.Label = Path-style access
+S3Meta.PathStyle.Description = Use path-style bucket access (required for some S3-compatible endpoints)
+S3Meta.CredentialsFile.Label = Credentials file
+S3Meta.CredentialsFile.Description = Path to AWS credentials file (overrides default ~/.aws/credentials)
+S3Meta.ProfileName.Label = Profile name
+S3Meta.ProfileName.Description = AWS profile name (when using credentials file)
+S3Meta.PartSize.Label = Part size
+S3Meta.PartSize.Description = Multipart upload part size (e.g. 5MB)
+S3Meta.CacheTtl.Label = Cache TTL (seconds)
+S3Meta.CacheTtl.Description = How long to cache folder listing results (in seconds, default 5)
+S3Meta.AuthType.Label = Authentication
+S3Meta.AuthType.Default = Default (environment / instance profile)
+S3Meta.AuthType.AccessKeys = Access key and secret
+S3Meta.AuthType.CredentialsFile = Credentials file / profile
diff --git a/plugins/tech/aws/src/main/resources/s3.svg b/plugins/tech/aws/src/main/resources/s3.svg
new file mode 100644
index 0000000000..3f63be51fa
--- /dev/null
+++ b/plugins/tech/aws/src/main/resources/s3.svg
@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="428" height="512" viewBox="0 0 428 512">
+ <defs>
+ <style>
+ .cls-1 {
+ fill: #e25444;
+ }
+
+ .cls-1, .cls-2, .cls-3 {
+ fill-rule: evenodd;
+ }
+
+ .cls-2 {
+ fill: #7b1d13;
+ }
+
+ .cls-3 {
+ fill: #58150d;
+ }
+ </style>
+ </defs>
+ <path class="cls-1" d="M378,99L295,257l83,158,34-19V118Z"/>
+ <path class="cls-2" d="M378,99L212,118,127.5,257,212,396l166,19V99Z"/>
+ <path class="cls-3" d="M43,99L16,111V403l27,12L212,257Z"/>
+ <path class="cls-1" d="M42.637,98.667l169.587,47.111V372.444L42.637,415.111V98.667Z"/>
+ <path class="cls-3" d="M212.313,170.667l-72.008-11.556,72.008-81.778,71.83,81.778Z"/>
+ <path class="cls-3" d="M284.143,159.111l-71.919,11.733-71.919-11.733V77.333"/>
+ <path class="cls-3" d="M212.313,342.222l-72.008,13.334,72.008,70.222,71.83-70.222Z"/>
+ <path class="cls-2" d="M212,16L140,54V159l72.224-20.333Z"/>
+ <path class="cls-2" d="M212.224,196.444l-71.919,7.823V309.105l71.919,8.228V196.444Z"/>
+ <path class="cls-2" d="M212.224,373.333L140.305,355.3V458.363L212.224,496V373.333Z"/>
+ <path class="cls-1" d="M284.143,355.3l-71.919,18.038V496l71.919-37.637V355.3Z"/>
+ <path class="cls-1" d="M212.224,196.444l71.919,7.823V309.105l-71.919,8.228V196.444Z"/>
+ <path class="cls-1" d="M212,16l72,38V159l-72-20V16Z"/>
+</svg>
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/s3common/S3ListCacheTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/s3common/S3ListCacheTest.java
new file mode 100644
index 0000000000..3fbfd5087c
--- /dev/null
+++ b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/s3common/S3ListCacheTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.s3.s3common;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.commons.vfs2.FileType;
+import org.junit.jupiter.api.Test;
+
+class S3ListCacheTest {
+
+ @Test
+ void testDefaultTtl() {
+ assertEquals(10_000L, S3ListCache.DEFAULT_TTL_MS);
+ }
+
+ @Test
+ void testPutAndGet() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("path/to/file.txt", new S3ListCache.ChildInfo(FileType.FILE, 1024, Instant.now()));
+ entries.put("path/to/sub/", new S3ListCache.ChildInfo(FileType.FOLDER, 0, Instant.EPOCH));
+
+ cache.put("mybucket", "path/to/", entries);
+
+ S3ListCache.ChildInfo fileInfo = cache.get("mybucket", "path/to/", "path/to/file.txt");
+ assertNotNull(fileInfo);
+ assertEquals(FileType.FILE, fileInfo.type);
+ assertEquals(1024, fileInfo.size);
+
+ S3ListCache.ChildInfo folderInfo = cache.get("mybucket", "path/to/", "path/to/sub/");
+ assertNotNull(folderInfo);
+ assertEquals(FileType.FOLDER, folderInfo.type);
+ assertEquals(0, folderInfo.size);
+ }
+
+ @Test
+ void testGetMissingKey() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("path/to/file.txt", new S3ListCache.ChildInfo(FileType.FILE, 100, Instant.now()));
+ cache.put("mybucket", "path/to/", entries);
+
+ assertNull(cache.get("mybucket", "path/to/", "path/to/other.txt"));
+ }
+
+ @Test
+ void testGetMissingBucket() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ assertNull(cache.get("nonexistent", "path/", "path/file"));
+ }
+
+ @Test
+ void testExpiry() throws InterruptedException {
+ S3ListCache cache = new S3ListCache(50L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("key", new S3ListCache.ChildInfo(FileType.FILE, 10, Instant.now()));
+ cache.put("bucket", "prefix/", entries);
+
+ assertNotNull(cache.get("bucket", "prefix/", "key"));
+
+ Thread.sleep(100);
+
+ assertNull(cache.get("bucket", "prefix/", "key"));
+ }
+
+ @Test
+ void testInvalidate() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("key", new S3ListCache.ChildInfo(FileType.FILE, 10, Instant.now()));
+ cache.put("bucket", "prefix/", entries);
+
+ assertNotNull(cache.get("bucket", "prefix/", "key"));
+ cache.invalidate("bucket", "prefix/");
+ assertNull(cache.get("bucket", "prefix/", "key"));
+ }
+
+ @Test
+ void testInvalidateParentOf() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("dir/file.txt", new S3ListCache.ChildInfo(FileType.FILE, 10, Instant.now()));
+ cache.put("bucket", "dir/", entries);
+
+ assertNotNull(cache.get("bucket", "dir/", "dir/file.txt"));
+ cache.invalidateParentOf("bucket", "dir/file.txt");
+ assertNull(cache.get("bucket", "dir/", "dir/file.txt"));
+ }
+
+ @Test
+ void testPutEmptyMap() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ cache.put("bucket", "prefix/", new LinkedHashMap<>());
+ assertNull(cache.get("bucket", "prefix/", "anything"));
+ }
+
+ @Test
+ void testPutNull() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ cache.put("bucket", "prefix/", null);
+ assertNull(cache.get("bucket", "prefix/", "anything"));
+ }
+
+ @Test
+ void testParentPrefix() {
+ assertEquals("path/to/", S3ListCache.parentPrefix("path/to/file"));
+ assertEquals("", S3ListCache.parentPrefix("file"));
+ assertEquals("", S3ListCache.parentPrefix(""));
+ assertEquals("", S3ListCache.parentPrefix(null));
+ assertEquals("a/b/", S3ListCache.parentPrefix("a/b/c"));
+ }
+
+ @Test
+ void testCustomTtl() {
+ S3ListCache cache = new S3ListCache(5_000L);
+ Map<String, S3ListCache.ChildInfo> entries = new LinkedHashMap<>();
+ entries.put("key", new S3ListCache.ChildInfo(FileType.FILE, 10, Instant.now()));
+ cache.put("bucket", "prefix/", entries);
+ assertNotNull(cache.get("bucket", "prefix/", "key"));
+ }
+
+ @Test
+ void testDifferentBucketsSamePrefix() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ Map<String, S3ListCache.ChildInfo> entries1 = new LinkedHashMap<>();
+ entries1.put("prefix/a", new S3ListCache.ChildInfo(FileType.FILE, 10, Instant.now()));
+ Map<String, S3ListCache.ChildInfo> entries2 = new LinkedHashMap<>();
+ entries2.put("prefix/b", new S3ListCache.ChildInfo(FileType.FILE, 20, Instant.now()));
+
+ cache.put("bucket1", "prefix/", entries1);
+ cache.put("bucket2", "prefix/", entries2);
+
+ assertNotNull(cache.get("bucket1", "prefix/", "prefix/a"));
+ assertNull(cache.get("bucket1", "prefix/", "prefix/b"));
+ assertNotNull(cache.get("bucket2", "prefix/", "prefix/b"));
+ assertNull(cache.get("bucket2", "prefix/", "prefix/a"));
+ }
+
+ @Test
+ void testInvalidateNonExistentEntry() {
+ S3ListCache cache = new S3ListCache(60_000L);
+ cache.invalidate("nonexistent", "prefix/");
+ cache.invalidateParentOf("nonexistent", "some/key");
+ }
+
+ @Test
+ void testChildInfoFields() {
+ Instant now = Instant.now();
+ S3ListCache.ChildInfo info = new S3ListCache.ChildInfo(FileType.FILE, 42, now);
+ assertEquals(FileType.FILE, info.type);
+ assertEquals(42, info.size);
+ assertEquals(now, info.lastModified);
+ }
+}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileObjectTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileObjectTest.java
index 8d0922b283..e44de11a79 100644
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileObjectTest.java
+++ b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileObjectTest.java
@@ -16,36 +16,21 @@
*/
package org.apache.hop.vfs.s3.vfs;
-import static java.util.AbstractMap.SimpleEntry;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyString;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.Bucket;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.UploadPartResult;
import java.io.OutputStream;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -70,6 +55,22 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.Bucket;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
class S3FileObjectTest {
@@ -85,16 +86,15 @@ class S3FileObjectTest {
private S3FileObject s3FileObjectBucketSpy;
private S3FileObject s3FileObjectFileSpy;
private S3FileObject s3FileObjectSpyRoot;
- private AmazonS3 s3ServiceMock;
- private ObjectListing childObjectListing;
- private S3Object s3ObjectMock;
- private S3ObjectInputStream s3ObjectInputStream;
- private ObjectMetadata s3ObjectMetadata;
+ private S3Client s3ClientMock;
+ private ListObjectsV2Response listResponsePage1;
+ private ListObjectsV2Response listResponsePage2;
private final List<String> childObjectNameComp = new ArrayList<>();
private final List<String> childBucketNameListComp = new ArrayList<>();
private final long contentLength = 42;
private final String origKey = "some/key";
private final Date testDate = new Date();
+ private final Instant testInstant = testDate.toInstant();
@BeforeAll
static void initHop() throws Exception {
@@ -104,10 +104,7 @@ class S3FileObjectTest {
@BeforeEach
void setUp() throws Exception {
- s3ServiceMock = mock(AmazonS3.class);
- S3Object s3Object = new S3Object();
- s3Object.setKey(OBJECT_NAME);
- s3Object.setBucketName(BUCKET_NAME);
+ s3ClientMock = mock(S3Client.class);
filename = new S3FileName(SCHEME, BUCKET_NAME, BUCKET_NAME, FileType.FOLDER);
S3FileName rootFileName = new S3FileName(SCHEME, "", "", FileType.FOLDER);
@@ -138,63 +135,71 @@ class S3FileObjectTest {
S3FileObject s3FileObjectRoot = new S3FileObject(rootFileName, fileSystemSpy);
s3FileObjectSpyRoot = spy(s3FileObjectRoot);
- // specify the behaviour of S3 Service
- when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
- when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
- when(s3ServiceMock.listBuckets()).thenReturn(createBuckets());
-
- when(s3ServiceMock.doesBucketExistV2(BUCKET_NAME)).thenReturn(true);
-
- childObjectListing = mock(ObjectListing.class);
- when(childObjectListing.getObjectSummaries())
- .thenReturn(createObjectSummaries(0))
- .thenReturn(new ArrayList<>());
- when(childObjectListing.getCommonPrefixes())
- .thenReturn(new ArrayList<>())
- .thenReturn(createCommonPrefixes(3));
- when(childObjectListing.isTruncated()).thenReturn(true).thenReturn(false);
-
- when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
- when(s3ServiceMock.listObjects(anyString(), anyString())).thenReturn(childObjectListing);
- when(s3ServiceMock.listNextBatchOfObjects(any(ObjectListing.class)))
- .thenReturn(childObjectListing);
-
- s3ObjectMock = mock(S3Object.class);
- s3ObjectInputStream = mock(S3ObjectInputStream.class);
- s3ObjectMetadata = mock(ObjectMetadata.class);
- when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream);
- when(s3ServiceMock.getObjectMetadata(anyString(), anyString())).thenReturn(s3ObjectMetadata);
- when(s3ObjectMetadata.getContentLength()).thenReturn(contentLength);
- when(s3ObjectMetadata.getLastModified()).thenReturn(testDate);
- when(s3ServiceMock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
-
- when(fileSystemSpy.getS3Client()).thenReturn(s3ServiceMock);
+ when(fileSystemSpy.getS3Client()).thenReturn(s3ClientMock);
+
+ // listBuckets for root
+ when(s3ClientMock.listBuckets())
+ .thenReturn(ListBucketsResponse.builder().buckets(createBuckets()).build());
+
+ // listObjectsV2 for folder listing (two pages: contents then commonPrefixes)
+ listResponsePage1 =
+ ListObjectsV2Response.builder()
+ .contents(createContents(0))
+ .commonPrefixes(new ArrayList<>())
+ .nextContinuationToken("token1")
+ .build();
+ listResponsePage2 =
+ ListObjectsV2Response.builder()
+ .contents(new ArrayList<>())
+ .commonPrefixes(createCommonPrefixes(3))
+ .nextContinuationToken(null)
+ .build();
+ when(s3ClientMock.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(listResponsePage1)
+ .thenReturn(listResponsePage2);
+
+ // getObject / headObject for file metadata and stream
+ ResponseInputStream<GetObjectResponse> mockStream = mock(ResponseInputStream.class);
+ GetObjectResponse getResponse =
+ GetObjectResponse.builder().contentLength(contentLength).lastModified(testInstant).build();
+ lenient().when(mockStream.response()).thenReturn(getResponse);
+ lenient().when(s3ClientMock.getObject(any(GetObjectRequest.class))).thenReturn(mockStream);
+
+ software.amazon.awssdk.services.s3.model.HeadObjectResponse headResponse =
+ software.amazon.awssdk.services.s3.model.HeadObjectResponse.builder()
+ .contentLength(contentLength)
+ .lastModified(testInstant)
+ .build();
+ lenient()
+ .when(
+ s3ClientMock.headObject(
+ any(software.amazon.awssdk.services.s3.model.HeadObjectRequest.class)))
+ .thenReturn(headResponse);
}
@Test
- void testGetS3Object() throws Exception {
- when(s3ServiceMock.getObject(anyString(), anyString())).thenReturn(new S3Object());
- S3FileObject s3FileObject = new S3FileObject(filename, fileSystemSpy);
- S3Object s3Object = s3FileObject.getS3Object();
- assertNotNull(s3Object);
+ void testGetInputStream() throws Exception {
+ assertNotNull(s3FileObjectBucketSpy.getInputStream());
}
@Test
void testGetS3BucketName() {
filename = new S3FileName(SCHEME, BUCKET_NAME, "", FileType.FOLDER);
when(s3FileObjectBucketSpy.getName()).thenReturn(filename);
- s3FileObjectBucketSpy.getS3BucketName();
+ s3FileObjectBucketSpy.getBucketName();
}
@Test
void testDoGetOutputStream() throws Exception {
- InitiateMultipartUploadResult initResponse = mock(InitiateMultipartUploadResult.class);
- when(initResponse.getUploadId()).thenReturn("foo");
- when(s3ServiceMock.initiateMultipartUpload(any())).thenReturn(initResponse);
- UploadPartResult uploadPartResult = mock(UploadPartResult.class);
- PartETag tag = mock(PartETag.class);
- when(s3ServiceMock.uploadPart(any())).thenReturn(uploadPartResult);
- when(uploadPartResult.getPartETag()).thenReturn(tag);
+ CreateMultipartUploadResponse createResponse =
+ CreateMultipartUploadResponse.builder().uploadId("foo").build();
+ when(s3ClientMock.createMultipartUpload(any(CreateMultipartUploadRequest.class)))
+ .thenReturn(createResponse);
+ UploadPartResponse uploadPartResponse = UploadPartResponse.builder().eTag("etag").build();
+ when(s3ClientMock.uploadPart(
+ any(software.amazon.awssdk.services.s3.model.UploadPartRequest.class),
+ any(software.amazon.awssdk.core.sync.RequestBody.class)))
+ .thenReturn(uploadPartResponse);
assertNotNull(s3FileObjectBucketSpy.doGetOutputStream(false));
OutputStream out = s3FileObjectBucketSpy.doGetOutputStream(true);
@@ -202,9 +207,13 @@ class S3FileObjectTest {
out.write(new byte[1024 * 1024 * 6]); // 6MB
out.close();
- // check that property 's3.vfs.partSize' is less than [5MB, 6MB)
- verify(s3ServiceMock, times(2)).uploadPart(any());
- verify(s3ServiceMock, atMost(1)).completeMultipartUpload(any());
+ verify(s3ClientMock, times(2))
+ .uploadPart(
+ any(software.amazon.awssdk.services.s3.model.UploadPartRequest.class),
+ any(software.amazon.awssdk.core.sync.RequestBody.class));
+ verify(s3ClientMock, atMost(1))
+ .completeMultipartUpload(
+ any(software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest.class));
}
@Test
@@ -234,11 +243,11 @@ class S3FileObjectTest {
SCHEME, BUCKET_NAME, BUCKET_NAME + "/" + origKey, FileType.IMAGINARY),
fileSystemSpy));
notRootBucket.createFolder();
- ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor =
- ArgumentCaptor.forClass(PutObjectRequest.class);
- verify(s3ServiceMock).putObject(putObjectRequestArgumentCaptor.capture());
- assertEquals(BUCKET_NAME, putObjectRequestArgumentCaptor.getValue().getBucketName());
- assertEquals("some/key/", putObjectRequestArgumentCaptor.getValue().getKey());
+ ArgumentCaptor<PutObjectRequest> putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class);
+ verify(s3ClientMock)
+ .putObject(putCaptor.capture(), any(software.amazon.awssdk.core.sync.RequestBody.class));
+ assertEquals(BUCKET_NAME, putCaptor.getValue().bucket());
+ assertEquals("some/key/", putCaptor.getValue().key());
}
@Test
@@ -251,18 +260,21 @@ class S3FileObjectTest {
@Test
void testCanRenameToNullFile() {
- // This is a bug / weakness in VFS itself
assertThrows(NullPointerException.class, () -> s3FileObjectBucketSpy.canRenameTo(null));
}
@Test
void testDoDelete() throws Exception {
+ when(s3ClientMock.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(
+ ListObjectsV2Response.builder()
+ .contents(createContents(0))
+ .nextContinuationToken(null)
+ .build());
fileSystemSpy.init();
s3FileObjectBucketSpy.doDelete();
- verify(s3ServiceMock).deleteObject("bucket3", "key0");
- verify(s3ServiceMock).deleteObject("bucket3", "key1");
- verify(s3ServiceMock).deleteObject("bucket3", "key2");
- verify(s3ServiceMock).deleteObject("bucket3", "");
+ verify(s3ClientMock, times(4))
+ .deleteObject(any(software.amazon.awssdk.services.s3.model.DeleteObjectRequest.class));
}
@Test
@@ -273,18 +285,15 @@ class S3FileObjectTest {
new S3FileName(
SCHEME, someNewBucketName, someNewBucketName + "/" + someNewKey, FileType.FILE);
S3FileObject newFile = new S3FileObject(newFileName, fileSystemSpy);
- ArgumentCaptor<CopyObjectRequest> copyObjectRequestArgumentCaptor =
- ArgumentCaptor.forClass(CopyObjectRequest.class);
- when(s3ServiceMock.doesBucketExistV2(someNewBucketName)).thenReturn(true);
+ ArgumentCaptor<CopyObjectRequest> copyCaptor = ArgumentCaptor.forClass(CopyObjectRequest.class);
s3FileObjectFileSpy.doAttach();
s3FileObjectFileSpy.moveTo(newFile);
- verify(s3ServiceMock).copyObject(copyObjectRequestArgumentCaptor.capture());
- assertEquals(
- someNewBucketName, copyObjectRequestArgumentCaptor.getValue().getDestinationBucketName());
- assertEquals(someNewKey, copyObjectRequestArgumentCaptor.getValue().getDestinationKey());
- assertEquals(BUCKET_NAME, copyObjectRequestArgumentCaptor.getValue().getSourceBucketName());
- assertEquals(origKey, copyObjectRequestArgumentCaptor.getValue().getSourceKey());
+ verify(s3ClientMock).copyObject(copyCaptor.capture());
+ assertEquals(someNewBucketName, copyCaptor.getValue().destinationBucket());
+ assertEquals(someNewKey, copyCaptor.getValue().destinationKey());
+ assertEquals(BUCKET_NAME, copyCaptor.getValue().sourceBucket());
+ assertEquals(origKey, copyCaptor.getValue().sourceKey());
}
@Test
@@ -299,7 +308,7 @@ class S3FileObjectTest {
FileObject[] children = s3FileObjectBucketSpy.getChildren();
assertEquals(6, children.length);
List<String> childNameArray =
- Arrays.asList(children).stream()
+ Arrays.stream(children)
.map(child -> child.getName().getPath())
.collect(Collectors.toList());
assertEquals(childObjectNameComp, childNameArray);
@@ -311,95 +320,116 @@ class S3FileObjectTest {
FileObject[] children = s3FileObjectSpyRoot.getChildren();
assertEquals(4, children.length);
List<String> childNameArray =
- Arrays.asList(children).stream()
+ Arrays.stream(children)
.map(child -> child.getName().getPath())
.collect(Collectors.toList());
assertEquals(childBucketNameListComp, childNameArray);
}
@Test
- void testFixFilePathToFile() {
- String bucketName = "s3:/";
- String key = "bucketName/some/key/path";
- SimpleEntry<String, String> newPath = s3FileObjectBucketSpy.fixFilePath(key, bucketName);
- assertEquals("bucketName", newPath.getValue());
- assertEquals("some/key/path", newPath.getKey());
+ void testHandleAttachException() throws Exception {
+ when(s3ClientMock.headObject(
+ any(software.amazon.awssdk.services.s3.model.HeadObjectRequest.class)))
+ .thenThrow(
+ (S3Exception)
+ S3Exception.builder()
+ .message("NoSuchKey")
+ .awsErrorDetails(AwsErrorDetails.builder().errorCode("NoSuchKey").build())
+ .build());
+ when(s3ClientMock.getObject(any(GetObjectRequest.class)))
+ .thenThrow(
+ (S3Exception)
+ S3Exception.builder()
+ .message("NoSuchKey")
+ .awsErrorDetails(AwsErrorDetails.builder().errorCode("NoSuchKey").build())
+ .build());
+ when(s3ClientMock.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(
+ ListObjectsV2Response.builder()
+ .contents(new ArrayList<>())
+ .commonPrefixes(createCommonPrefixes(1))
+ .build());
+ s3FileObjectFileSpy.doAttach();
+ assertEquals(FileType.FOLDER, s3FileObjectFileSpy.getType());
}
@Test
- void testFixFilePathToFolder() {
- String bucketName = "s3:/";
- String key = "bucketName";
- SimpleEntry<String, String> newPath = s3FileObjectBucketSpy.fixFilePath(key, bucketName);
- assertEquals("bucketName", newPath.getValue());
- assertEquals("", newPath.getKey());
+ void testDoAttachCacheHit() throws Exception {
+ java.util.Map<String, org.apache.hop.vfs.s3.s3common.S3ListCache.ChildInfo> entries =
+ new java.util.LinkedHashMap<>();
+ entries.put(
+ origKey,
+ new org.apache.hop.vfs.s3.s3common.S3ListCache.ChildInfo(
+ FileType.FILE, 999, Instant.now()));
+ fileSystemSpy.putListCache(BUCKET_NAME, "some/", entries);
+
+ s3FileObjectFileSpy.doAttach();
+ assertEquals(FileType.FILE, s3FileObjectFileSpy.getType());
}
@Test
- void testHandleAttachException() throws FileSystemException {
- String testKey = BUCKET_NAME + "/" + origKey;
- String testBucket = "badBucketName";
- AmazonS3Exception exception = new AmazonS3Exception("NoSuchKey");
-
- // test the case where the folder exists and contains things; no exception should be thrown
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey + "/")).thenThrow(exception);
- try {
- s3FileObjectFileSpy.handleAttachException(testKey, testBucket);
- } catch (FileSystemException e) {
- fail("Caught exception " + e.getMessage());
- }
+ void testDoAttachCacheHitFolder() throws Exception {
+ java.util.Map<String, org.apache.hop.vfs.s3.s3common.S3ListCache.ChildInfo> entries =
+ new java.util.LinkedHashMap<>();
+ entries.put(
+ origKey + "/",
+ new org.apache.hop.vfs.s3.s3common.S3ListCache.ChildInfo(
+ FileType.FOLDER, 0, Instant.EPOCH));
+ fileSystemSpy.putListCache(BUCKET_NAME, "some/", entries);
+
+ s3FileObjectFileSpy.doAttach();
assertEquals(FileType.FOLDER, s3FileObjectFileSpy.getType());
}
@Test
- void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
- String testKey = BUCKET_NAME + "/" + origKey;
- String testBucket = "badBucketName";
- AmazonS3Exception exception = new AmazonS3Exception("NoSuchKey");
- exception.setErrorCode("NoSuchKey");
-
- // test the case where the folder exists and contains things; no exception should be thrown
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey + "/")).thenThrow(exception);
- childObjectListing = mock(ObjectListing.class);
- when(childObjectListing.getObjectSummaries()).thenReturn(new ArrayList<>());
- when(childObjectListing.getCommonPrefixes()).thenReturn(new ArrayList<>());
- when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
- try {
- s3FileObjectFileSpy.handleAttachException(testKey, testBucket);
- } catch (FileSystemException e) {
- fail("Caught exception " + e.getMessage());
- }
+ void testHandleAttachExceptionEmptyFolder() throws Exception {
+ S3Exception exception =
+ (S3Exception)
+ S3Exception.builder()
+ .message("NoSuchKey")
+ .awsErrorDetails(AwsErrorDetails.builder().errorCode("NoSuchKey").build())
+ .build();
+ when(s3ClientMock.headObject(
+ any(software.amazon.awssdk.services.s3.model.HeadObjectRequest.class)))
+ .thenThrow(exception);
+ when(s3ClientMock.getObject(any(GetObjectRequest.class))).thenThrow(exception);
+ when(s3ClientMock.listObjectsV2(any(ListObjectsV2Request.class)))
+ .thenReturn(
+ ListObjectsV2Response.builder()
+ .contents(new ArrayList<>())
+ .commonPrefixes(new ArrayList<>())
+ .build());
+ s3FileObjectFileSpy.doAttach();
assertEquals(FileType.IMAGINARY, s3FileObjectFileSpy.getType());
}
private List<Bucket> createBuckets() {
List<Bucket> buckets = new ArrayList<>();
- buckets.add(new Bucket("bucket1"));
- buckets.add(new Bucket("bucket2"));
- buckets.add(new Bucket("bucket3"));
- buckets.add(new Bucket("bucket4"));
+ buckets.add(Bucket.builder().name("bucket1").build());
+ buckets.add(Bucket.builder().name("bucket2").build());
+ buckets.add(Bucket.builder().name("bucket3").build());
+ buckets.add(Bucket.builder().name("bucket4").build());
childBucketNameListComp.addAll(
- buckets.stream().map(bucket -> "/" + bucket.getName()).collect(Collectors.toList()));
+ buckets.stream().map(b -> "/" + b.name()).collect(Collectors.toList()));
return buckets;
}
- private List<S3ObjectSummary> createObjectSummaries(int startnum) {
- List<S3ObjectSummary> summaries = new ArrayList<>();
- for (int i = startnum; i < startnum + 3; i++) {
- S3ObjectSummary summary = new S3ObjectSummary();
- summary.setBucketName(BUCKET_NAME);
- summary.setKey("key" + i);
- summaries.add(summary);
- childObjectNameComp.add(BUCKET_NAME + "/" + "key" + i);
+ private List<software.amazon.awssdk.services.s3.model.S3Object> createContents(int startNum) {
+ List<software.amazon.awssdk.services.s3.model.S3Object> list = new ArrayList<>();
+ for (int i = startNum; i < startNum + 3; i++) {
+ String k = "key" + i;
+ list.add(software.amazon.awssdk.services.s3.model.S3Object.builder().key(k).size(0L).build());
+ childObjectNameComp.add(BUCKET_NAME + "/" + k);
}
- return summaries;
+ return list;
}
- private List<String> createCommonPrefixes(int startnum) {
- List<String> prefixes = new ArrayList<>();
- for (int i = startnum; i < startnum + 3; i++) {
- prefixes.add("key" + i);
- childObjectNameComp.add(BUCKET_NAME + "/" + "key" + i);
+ private List<CommonPrefix> createCommonPrefixes(int count) {
+ List<CommonPrefix> prefixes = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ String p = "key" + i;
+ prefixes.add(CommonPrefix.builder().prefix(p).build());
+ childObjectNameComp.add(BUCKET_NAME + "/" + p);
}
return prefixes;
}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileProviderTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileProviderTest.java
index bac02aa4a8..4dfad3960e 100644
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileProviderTest.java
+++ b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileProviderTest.java
@@ -20,7 +20,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileSystem;
import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.vfs.s3.metadata.S3AuthType;
+import org.apache.hop.vfs.s3.metadata.S3Meta;
import org.apache.hop.vfs.s3.s3.vfs.S3FileProvider;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -29,16 +34,137 @@ import org.junit.jupiter.api.Test;
class S3FileProviderTest {
S3FileProvider provider;
+ FileName fileName;
+ IVariables variables;
@BeforeEach
void setUp() {
provider = new S3FileProvider();
+ fileName = mock(FileName.class);
+ variables = new Variables();
}
@Test
void testDoCreateFileSystem() {
- FileName fileName = mock(FileName.class);
FileSystemOptions options = new FileSystemOptions();
assertNotNull(provider.doCreateFileSystem(fileName, options));
}
+
+ @Test
+ void testDoCreateFileSystemWithAccessKeys() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.ACCESS_KEYS.name());
+ meta.setAccessKey("myAccessKey");
+ meta.setSecretKey("mySecretKey");
+ meta.setSessionToken("myToken");
+ meta.setRegion("us-west-2");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemWithCredentialsFile() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.CREDENTIALS_FILE.name());
+ meta.setCredentialsFile("/some/path/credentials");
+ meta.setProfileName("myprofile");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemWithDefaultAuthAndProfile() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.DEFAULT.name());
+ meta.setProfileName("minio");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemClearsFieldsForAccessKeys() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.ACCESS_KEYS.name());
+ meta.setAccessKey("key");
+ meta.setSecretKey("secret");
+ meta.setProfileName(null);
+ meta.setCredentialsFile(null);
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemWithEndpoint() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.DEFAULT.name());
+ meta.setEndpoint("http://localhost:9000");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemCacheTtl() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.DEFAULT.name());
+ meta.setCacheTtlSeconds("15");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemInferredAuthType() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(null);
+ meta.setAccessKey("key");
+ meta.setSecretKey("secret");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemInferredAuthTypeCredentials() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(null);
+ meta.setCredentialsFile("/path/to/credentials");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemInferredAuthTypeDefault() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(null);
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testDoCreateFileSystemWithPathStyleAccess() {
+ S3Meta meta = new S3Meta();
+ meta.setAuthenticationType(S3AuthType.DEFAULT.name());
+ meta.setPathStyleAccess(true);
+ meta.setEndpoint("http://localhost:9000");
+
+ S3FileProvider namedProvider = new S3FileProvider(variables, meta);
+ FileSystem fs = namedProvider.doCreateFileSystem(fileName, null);
+ assertNotNull(fs);
+ }
}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileSystemTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileSystemTest.java
index 929728cc80..967b456d51 100644
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileSystemTest.java
+++ b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileSystemTest.java
@@ -32,6 +32,7 @@ import org.apache.hop.core.util.StorageUnitConverter;
import org.apache.hop.vfs.s3.s3.vfs.S3FileName;
import org.apache.hop.vfs.s3.s3.vfs.S3FileProvider;
import org.apache.hop.vfs.s3.s3.vfs.S3FileSystem;
+import org.apache.hop.vfs.s3.s3common.S3CommonFileSystemConfigBuilder;
import org.apache.hop.vfs.s3.s3common.S3HopProperty;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -142,6 +143,38 @@ class S3FileSystemTest {
assertEquals(_10MBLong, s3FileSystem.convertToLong("10MB"));
}
+ @Test
+ void testCacheTtlFromOptions() {
+ FileSystemOptions options = new FileSystemOptions();
+ S3CommonFileSystemConfigBuilder config = new S3CommonFileSystemConfigBuilder(options);
+ config.setCacheTtlSeconds("15");
+
+ S3FileSystem fs = new S3FileSystem(fileName, options);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testCacheTtlDefaultWhenNotSet() {
+ S3FileSystem fs = new S3FileSystem(fileName, new FileSystemOptions());
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testCacheTtlWithNullOptions() {
+ S3FileSystem fs = new S3FileSystem(fileName, null);
+ assertNotNull(fs);
+ }
+
+ @Test
+ void testCacheTtlWithInvalidValue() {
+ FileSystemOptions options = new FileSystemOptions();
+ S3CommonFileSystemConfigBuilder config = new S3CommonFileSystemConfigBuilder(options);
+ config.setCacheTtlSeconds("notanumber");
+
+ S3FileSystem fs = new S3FileSystem(fileName, options);
+ assertNotNull(fs);
+ }
+
public S3FileSystem getTestInstance() {
FileName rootName = mock(FileName.class);
FileSystemOptions fileSystemOptions = new FileSystemOptions();
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameParserTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameParserTest.java
deleted file mode 100644
index cbafb01375..0000000000
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameParserTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.vfs.s3.vfs;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.mockito.Mockito.mock;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.provider.FileNameParser;
-import org.apache.commons.vfs2.provider.VfsComponentContext;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileName;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileNameParser;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-/** Unit tests for S3FileNameParser */
-class S3NFileNameParserTest {
-
- FileNameParser parser;
-
- @BeforeEach
- void setUp() {
- parser = S3NFileNameParser.getInstance();
- }
-
- @Test
- void testParseUri() throws Exception {
- VfsComponentContext context = mock(VfsComponentContext.class);
- FileName fileName = mock(FileName.class);
- String uri = "s3n://bucket/file";
- FileName noBaseFile = parser.parseUri(context, null, uri);
- assertNotNull(noBaseFile);
- assertEquals("bucket", ((S3NFileName) noBaseFile).getBucketId());
- FileName withBaseFile = parser.parseUri(context, fileName, uri);
- assertNotNull(withBaseFile);
- assertEquals("bucket", ((S3NFileName) withBaseFile).getBucketId());
-
- // assumption is that the whole URL is valid until it comes time to resolve to S3 objects
- uri = "s3n://s3n/bucket/file";
- withBaseFile = parser.parseUri(context, fileName, uri);
- assertEquals("s3n", ((S3NFileName) withBaseFile).getBucketId());
- }
-}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameTest.java
deleted file mode 100644
index 6133bd8458..0000000000
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.vfs.s3.vfs;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.apache.commons.vfs2.FileType;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileName;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-class S3NFileNameTest {
-
- private S3NFileName fileName = null;
-
- public static final String HOST = "S3";
- public static final String SCHEME = "s3n";
- public static final int PORT = 843;
-
- @BeforeEach
- void setup() {
- fileName = new S3NFileName(SCHEME, "", "", FileType.FOLDER);
- }
-
- @Test
- void testGetURI() {
- String expected = buildS3URL("/");
- assertEquals(expected, fileName.getURI());
- }
-
- @Test
- void testCreateName() {
- assertEquals(
- "s3n:///path/to/my/file", fileName.createName("/path/to/my/file", FileType.FILE).getURI());
- }
-
- @Test
- void testAppendRootUriWithNonDefaultPort() {
- fileName = new S3NFileName(SCHEME, "", "FooFolder", FileType.FOLDER);
- String expectedUri = SCHEME + "://" + "FooFolder";
- assertEquals(expectedUri, fileName.getURI());
-
- fileName = new S3NFileName(SCHEME, "FooBucket", "/FooBucket/FooFolder", FileType.FOLDER);
- expectedUri = SCHEME + "://FooBucket/FooBucket/" + "FooFolder";
- assertEquals(expectedUri, fileName.getURI());
- }
-
- public static String buildS3URL(String path) {
- return SCHEME + "://" + path;
- }
-}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileObjectTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileObjectTest.java
deleted file mode 100644
index 9fec59f9d2..0000000000
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileObjectTest.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.vfs.s3.vfs;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.Bucket;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.apache.commons.vfs2.CacheStrategy;
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileObject;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.FilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.apache.commons.vfs2.provider.VfsComponentContext;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileName;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileObject;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileSystem;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.mockito.ArgumentCaptor;
-
-class S3NFileObjectTest {
-
- public static final String HOST = "S3";
- public static final String SCHEME = "s3n";
- public static final int PORT = 843;
-
- public static final String BUCKET_NAME = "bucket3";
- public static final String OBJECT_NAME = "obj";
-
- private S3NFileName filename;
- private S3NFileSystem fileSystemSpy;
- private S3NFileObject s3FileObjectBucketSpy;
- private S3NFileObject s3FileObjectFileSpy;
- private S3NFileObject s3FileObjectSpyRoot;
- private AmazonS3 s3ServiceMock;
- private static final String S3VFS_USE_TEMPORARY_FILE_ON_UPLOAD_DATA =
- "s3.vfs.useTempFileOnUploadData";
- private ObjectListing childObjectListing;
- private S3Object s3ObjectMock;
- private S3ObjectInputStream s3ObjectInputStream;
- private ObjectMetadata s3ObjectMetadata;
- private final List<String> childObjectNameComp = new ArrayList<>();
- private final List<String> childBucketNameListComp = new ArrayList<>();
- private final long contentLength = 42;
- private final String origKey = "some/key";
- private final Date testDate = new Date();
-
- @BeforeEach
- void setUp() throws Exception {
-
- s3ServiceMock = mock(AmazonS3.class);
- S3Object s3Object = new S3Object();
- s3Object.setKey(OBJECT_NAME);
- s3Object.setBucketName(BUCKET_NAME);
-
- filename = new S3NFileName(SCHEME, BUCKET_NAME, "/" + BUCKET_NAME, FileType.FOLDER);
- S3NFileName rootFileName = new S3NFileName(SCHEME, BUCKET_NAME, "", FileType.FOLDER);
- S3NFileSystem fileSystem = new S3NFileSystem(rootFileName, new FileSystemOptions());
- fileSystemSpy = spy(fileSystem);
- VfsComponentContext context = mock(VfsComponentContext.class);
- final DefaultFileSystemManager fsm = new DefaultFileSystemManager();
- FilesCache cache = mock(FilesCache.class);
- fsm.setFilesCache(cache);
- fsm.setCacheStrategy(CacheStrategy.ON_RESOLVE);
- when(context.getFileSystemManager()).thenReturn(fsm);
- fileSystemSpy.setContext(context);
-
- S3NFileObject s3FileObject = new S3NFileObject(filename, fileSystemSpy);
- s3FileObjectBucketSpy = spy(s3FileObject);
-
- s3FileObjectFileSpy =
- spy(
- new S3NFileObject(
- new S3NFileName(
- SCHEME, BUCKET_NAME, "/" + BUCKET_NAME + "/" + origKey, FileType.IMAGINARY),
- fileSystemSpy));
-
- S3NFileObject s3FileObjectRoot = new S3NFileObject(rootFileName, fileSystemSpy);
- s3FileObjectSpyRoot = spy(s3FileObjectRoot);
-
- // specify the behaviour of S3 Service
- when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
- when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
- when(s3ServiceMock.listBuckets()).thenReturn(createBuckets());
-
- when(s3ServiceMock.doesBucketExistV2(BUCKET_NAME)).thenReturn(true);
-
- childObjectListing = mock(ObjectListing.class);
- when(childObjectListing.getObjectSummaries())
- .thenReturn(createObjectSummaries(0))
- .thenReturn(new ArrayList<>());
- when(childObjectListing.getCommonPrefixes())
- .thenReturn(new ArrayList<>())
- .thenReturn(createCommonPrefixes(3));
- when(childObjectListing.isTruncated()).thenReturn(true).thenReturn(false);
-
- when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
- when(s3ServiceMock.listObjects(anyString(), anyString())).thenReturn(childObjectListing);
- when(s3ServiceMock.listNextBatchOfObjects(any(ObjectListing.class)))
- .thenReturn(childObjectListing);
-
- s3ObjectMock = mock(S3Object.class);
- s3ObjectInputStream = mock(S3ObjectInputStream.class);
- s3ObjectMetadata = mock(ObjectMetadata.class);
- when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream);
- when(s3ServiceMock.getObjectMetadata(anyString(), anyString())).thenReturn(s3ObjectMetadata);
- when(s3ObjectMetadata.getContentLength()).thenReturn(contentLength);
- when(s3ObjectMetadata.getLastModified()).thenReturn(testDate);
- when(s3ServiceMock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
-
- when(fileSystemSpy.getS3Client()).thenReturn(s3ServiceMock);
- }
-
- @Test
- void testGetS3Object() {
- when(s3ServiceMock.getObject(anyString(), anyString())).thenReturn(new S3Object());
- S3NFileObject s3FileObject = new S3NFileObject(filename, fileSystemSpy);
- S3Object s3Object = s3FileObject.getS3Object();
- assertNotNull(s3Object);
- }
-
- @Test
- void testDoGetInputStream() throws Exception {
- assertNotNull(s3FileObjectBucketSpy.getInputStream());
- }
-
- @Test
- void testDoGetTypeImaginary() throws Exception {
- assertEquals(FileType.IMAGINARY, s3FileObjectFileSpy.getType());
- }
-
- @Test
- void testDoGetTypeFolder() throws Exception {
- FileName mockFile = mock(FileName.class);
- when(s3FileObjectBucketSpy.getName()).thenReturn(mockFile);
- when(mockFile.getPath()).thenReturn(S3NFileObject.DELIMITER);
- assertEquals(FileType.FOLDER, s3FileObjectBucketSpy.getType());
- }
-
- @Test
- void testDoCreateFolder() throws Exception {
- S3NFileObject notRootBucket =
- spy(
- new S3NFileObject(
- new S3NFileName(
- SCHEME, BUCKET_NAME, "/" + BUCKET_NAME + "/" + origKey, FileType.IMAGINARY),
- fileSystemSpy));
- notRootBucket.createFolder();
- ArgumentCaptor<PutObjectRequest> putObjectRequestArgumentCaptor =
- ArgumentCaptor.forClass(PutObjectRequest.class);
- verify(s3ServiceMock).putObject(putObjectRequestArgumentCaptor.capture());
- assertEquals(BUCKET_NAME, putObjectRequestArgumentCaptor.getValue().getBucketName());
- assertEquals("some/key/", putObjectRequestArgumentCaptor.getValue().getKey());
- }
-
- @Test
- void testCanRenameTo() throws Exception {
- FileObject newFile = mock(FileObject.class);
- assertFalse(s3FileObjectBucketSpy.canRenameTo(newFile));
- when(s3FileObjectBucketSpy.getType()).thenReturn(FileType.FOLDER);
- assertFalse(s3FileObjectBucketSpy.canRenameTo(newFile));
- }
-
- @Test
- void testCanRenameToNullFile() {
- // This is a bug / weakness in VFS itself
- assertThrows(NullPointerException.class, () -> s3FileObjectBucketSpy.canRenameTo(null));
- }
-
- @Test
- void testDoRename() throws Exception {
- String someNewBucketName = "someNewBucketName";
- String someNewKey = "some/newKey";
- S3NFileName newFileName =
- new S3NFileName(
- SCHEME, someNewBucketName, "/" + someNewBucketName + "/" + someNewKey, FileType.FILE);
- S3NFileObject newFile = new S3NFileObject(newFileName, fileSystemSpy);
- ArgumentCaptor<CopyObjectRequest> copyObjectRequestArgumentCaptor =
- ArgumentCaptor.forClass(CopyObjectRequest.class);
- when(s3ServiceMock.doesBucketExistV2(someNewBucketName)).thenReturn(true);
- s3FileObjectFileSpy.moveTo(newFile);
-
- verify(s3ServiceMock).copyObject(copyObjectRequestArgumentCaptor.capture());
- assertEquals(
- someNewBucketName, copyObjectRequestArgumentCaptor.getValue().getDestinationBucketName());
- assertEquals(someNewKey, copyObjectRequestArgumentCaptor.getValue().getDestinationKey());
- assertEquals(BUCKET_NAME, copyObjectRequestArgumentCaptor.getValue().getSourceBucketName());
- assertEquals(origKey, copyObjectRequestArgumentCaptor.getValue().getSourceKey());
- }
-
- @Test
- void testDoGetLastModifiedTime() throws Exception {
- s3FileObjectFileSpy.doAttach();
- assertEquals(testDate.getTime(), s3FileObjectFileSpy.doGetLastModifiedTime());
- }
-
- @Test
- void testListChildrenNotRoot() throws FileSystemException {
- fileSystemSpy.init();
- FileObject[] children = s3FileObjectBucketSpy.getChildren();
- assertEquals(6, children.length);
- List<String> childNameArray =
- Arrays.asList(children).stream()
- .map(child -> child.getName().getPath())
- .collect(Collectors.toList());
- assertEquals(childObjectNameComp, childNameArray);
- }
-
- @Test
- void testListChildrenRoot() throws FileSystemException {
- fileSystemSpy.init();
- FileObject[] children = s3FileObjectSpyRoot.getChildren();
- assertEquals(4, children.length);
- List<String> childNameArray =
- Arrays.asList(children).stream()
- .map(child -> child.getName().getPath())
- .collect(Collectors.toList());
- assertEquals(childBucketNameListComp, childNameArray);
- }
-
- @Test
- void testHandleAttachException() throws FileSystemException {
- AmazonS3Exception exception = new AmazonS3Exception("NoSuchKey");
- exception.setErrorCode("NoSuchKey");
- // test the case where the folder exists and contains things; no exception should be thrown
- when(s3ServiceMock.getObjectMetadata(BUCKET_NAME, origKey)).thenThrow(exception);
- try {
- s3FileObjectFileSpy.doAttach();
- } catch (Exception e) {
- fail("Caught exception " + e.getMessage());
- }
- assertEquals(FileType.FOLDER, s3FileObjectFileSpy.getType());
- }
-
- @Test
- void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
- AmazonS3Exception exception = new AmazonS3Exception("NoSuchKey");
- exception.setErrorCode("NoSuchKey");
-
- // test the case where the folder exists and contains things; no exception should be thrown
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey)).thenThrow(exception);
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey + "/")).thenThrow(exception);
- childObjectListing = mock(ObjectListing.class);
- when(childObjectListing.getObjectSummaries()).thenReturn(new ArrayList<>());
- when(childObjectListing.getCommonPrefixes()).thenReturn(new ArrayList<>());
- when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
- try {
- s3FileObjectFileSpy.doAttach();
- } catch (Exception e) {
- fail("Caught exception " + e.getMessage());
- }
- assertEquals(FileType.IMAGINARY, s3FileObjectFileSpy.getType());
- }
-
- @Test
- void testHandleAttachExceptionFileNotFound() throws FileSystemException {
- AmazonS3Exception notFoundException = new AmazonS3Exception("404 Not Found");
- notFoundException.setErrorCode("404 Not Found");
- AmazonS3Exception noSuchKeyException = new AmazonS3Exception("NoSuchKey");
- noSuchKeyException.setErrorCode("NoSuchKey");
-
- // test the case where the file is not found; no exception should be thrown
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey)).thenThrow(notFoundException);
- when(s3ServiceMock.getObject(BUCKET_NAME, origKey + "/")).thenThrow(noSuchKeyException);
- childObjectListing = mock(ObjectListing.class);
- when(childObjectListing.getObjectSummaries()).thenReturn(new ArrayList<>());
- when(childObjectListing.getCommonPrefixes()).thenReturn(new ArrayList<>());
- when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
- try {
- s3FileObjectFileSpy.doAttach();
- } catch (Exception e) {
- fail("Caught exception " + e.getMessage());
- }
- assertEquals(FileType.IMAGINARY, s3FileObjectFileSpy.getType());
- }
-
- private List<Bucket> createBuckets() {
- List<Bucket> buckets = new ArrayList<>();
- buckets.add(new Bucket("bucket1"));
- buckets.add(new Bucket("bucket2"));
- buckets.add(new Bucket("bucket3"));
- buckets.add(new Bucket("bucket4"));
- childBucketNameListComp.addAll(
- buckets.stream().map(bucket -> "/" + bucket.getName()).collect(Collectors.toList()));
- return buckets;
- }
-
- private List<S3ObjectSummary> createObjectSummaries(int startnum) {
- List<S3ObjectSummary> summaries = new ArrayList<>();
- for (int i = startnum; i < startnum + 3; i++) {
- S3ObjectSummary summary = new S3ObjectSummary();
- summary.setBucketName(BUCKET_NAME);
- summary.setKey("key" + i);
- summaries.add(summary);
- childObjectNameComp.add("/" + BUCKET_NAME + "/" + "key" + i);
- }
- return summaries;
- }
-
- private List<String> createCommonPrefixes(int startnum) {
- List<String> prefixes = new ArrayList<>();
- for (int i = startnum; i < startnum + 3; i++) {
- prefixes.add("key" + i);
- childObjectNameComp.add("/" + BUCKET_NAME + "/" + "key" + i);
- }
- return prefixes;
- }
-}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileProviderTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileProviderTest.java
deleted file mode 100644
index a57883815e..0000000000
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileProviderTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.vfs.s3.vfs;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.mockito.Mockito.mock;
-
-import org.apache.commons.vfs2.FileName;
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileProvider;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-/** Unit tests for S3FileProvider */
-class S3NFileProviderTest {
-
- S3NFileProvider provider;
-
- @BeforeEach
- void setUp() {
- provider = new S3NFileProvider();
- }
-
- @Test
- void testDoCreateFileSystem() throws Exception {
- FileName fileName = mock(FileName.class);
- FileSystemOptions options = new FileSystemOptions();
- assertNotNull(provider.doCreateFileSystem(fileName, options));
- }
-}
diff --git a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileSystemTest.java b/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileSystemTest.java
deleted file mode 100644
index 82b986d843..0000000000
--- a/plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileSystemTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.vfs.s3.vfs;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.UserAuthenticationData;
-import org.apache.commons.vfs2.UserAuthenticator;
-import org.apache.commons.vfs2.impl.DefaultFileSystemConfigBuilder;
-import org.apache.hop.vfs.s3.s3.vfs.S3FileName;
-import org.apache.hop.vfs.s3.s3.vfs.S3FileProvider;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileName;
-import org.apache.hop.vfs.s3.s3n.vfs.S3NFileSystem;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-/** Unit tests for S3FileSystem */
-class S3NFileSystemTest {
-
- S3NFileSystem fileSystem;
- S3NFileName fileName;
-
- @BeforeEach
- void setUp() {
- fileName = new S3NFileName(S3FileNameTest.SCHEME, "/", "", FileType.FOLDER);
- fileSystem = new S3NFileSystem(fileName, new FileSystemOptions());
- }
-
- @Test
- void testCreateFile() throws Exception {
- assertNotNull(
- fileSystem.createFile(
- new S3FileName("s3n", "bucketName", "/bucketName/key", FileType.FILE)));
- }
-
- @Test
- void testGetS3Service() {
- assertNotNull(fileSystem.getS3Client());
-
- FileSystemOptions options = new FileSystemOptions();
- UserAuthenticator authenticator = mock(UserAuthenticator.class);
- UserAuthenticationData authData = mock(UserAuthenticationData.class);
- when(authenticator.requestAuthentication(S3FileProvider.AUTHENTICATOR_TYPES))
- .thenReturn(authData);
- when(authData.getData(UserAuthenticationData.USERNAME)).thenReturn("username".toCharArray());
- when(authData.getData(UserAuthenticationData.PASSWORD)).thenReturn("password".toCharArray());
- DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(options, authenticator);
-
- fileSystem = new S3NFileSystem(fileName, options);
- assertNotNull(fileSystem.getS3Client());
- }
-}
diff --git a/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMeta.java b/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMeta.java
index 95b0f5b420..0eaea17d0a 100644
--- a/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMeta.java
+++ b/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMeta.java
@@ -156,6 +156,7 @@ public class GetFileNamesMeta extends BaseTransformMeta<GetFileNames, GetFileNam
super(); // allocate BaseTransformMeta
filesList = new ArrayList<>();
filterItemList = new ArrayList<>();
+ raiseAnExceptionIfNoFile = true;
}
public String getRequiredFilesDesc(String tt) {
diff --git a/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMetaAdditionalTest.java b/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMetaAdditionalTest.java
index f0045457e1..71c476bc5f 100644
--- a/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMetaAdditionalTest.java
+++ b/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesMetaAdditionalTest.java
@@ -70,7 +70,7 @@ class GetFileNamesMetaAdditionalTest {
assertEquals("", meta.getDynamicWildcardField());
assertFalse(meta.isDynamicIncludeSubFolders());
assertEquals("", meta.getDynamicExcludeWildcardField());
- assertFalse(meta.isRaiseAnExceptionIfNoFile());
+ assertTrue(meta.isRaiseAnExceptionIfNoFile());
assertEquals(0, meta.getRowLimit());
// Check that default filter item is added
diff --git a/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesTransformTest.java b/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesTransformTest.java
index f404a4086a..8c08d206de 100644
--- a/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesTransformTest.java
+++ b/plugins/transforms/getfilenames/src/test/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesTransformTest.java
@@ -90,6 +90,7 @@ class GetFileNamesTransformTest {
// Setup meta for non-file field mode
GetFileNamesMeta meta = new GetFileNamesMeta();
meta.setFileField(false);
+ meta.setRaiseAnExceptionIfNoFile(false);
meta.setDefault();
// Add a file item to the list
@@ -124,6 +125,7 @@ class GetFileNamesTransformTest {
// Setup meta for file field mode
GetFileNamesMeta meta = new GetFileNamesMeta();
meta.setFileField(true);
+ meta.setRaiseAnExceptionIfNoFile(false);
meta.setDefault();
meta.setDynamicFilenameField("filename");