You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2023/06/14 01:18:07 UTC
[asterixdb] branch master updated: [ASTERIXDB-3196][CONF] Add cloud properties
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9f38f8df67 [ASTERIXDB-3196][CONF] Add cloud properties
9f38f8df67 is described below
commit 9f38f8df67daa1296fae7c9633d635c2b45adfb4
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Mon Jun 12 01:55:35 2023 +0300
[ASTERIXDB-3196][CONF] Add cloud properties
- user model changes: yes
- storage format changes: no
- interface changes: yes
Details:
- Add the following cloud properties to allow configuring cloud
storage which can be used for compute-storage separation:
-- cloud.storage.scheme
-- cloud.storage.bucket
-- cloud.storage.region
-- cloud.storage.prefix
-- cloud.storage.endpoint
-- cloud.storage.anonymous.auth
Change-Id: I1184818476c9b35bb521dc959033eeedd2a836c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17594
Reviewed-by: Murtadha Al Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/app/cc/CcApplicationContext.java | 8 ++
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 10 +-
.../src/main/resources/cc-cloud-storage.conf | 68 ------------
.../org/apache/asterix/api/common/CloudUtils.java | 11 +-
.../test/cloud_storage/CloudStorageTest.java | 2 +-
.../src/test/resources/cc-cloud-storage.conf | 7 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 6 ++
.../cluster_state_1_full.1.regexadm | 6 ++
.../cluster_state_1_less.1.regexadm | 6 ++
.../org/apache/asterix/cloud/CloudIOManager.java | 18 ++--
.../asterix/cloud/clients/CloudClientProvider.java | 27 ++---
.../clients/ICloudClientCredentialsProvider.java | 30 ------
.../apache/asterix/cloud/clients/ICredentials.java | 22 ----
.../cloud/clients/aws/s3/S3ClientConfig.java | 61 +++++++++++
.../cloud/clients/aws/s3/S3CloudClient.java | 40 +++----
.../aws/s3/credentials/FileCredentials.java | 74 -------------
.../clients/aws/s3/credentials/IS3Credentials.java | 44 --------
.../aws/s3/credentials/S3CredentialsProvider.java | 45 --------
.../aws/s3/credentials/S3MockCredentials.java | 52 ---------
.../storage/CloudStorageConfigurationProvider.java | 44 --------
.../storage/FileCloudStorageConfiguration.java | 49 ---------
.../cloud/storage/ICloudStorageConfiguration.java | 32 ------
.../storage/MockCloudStorageConfiguration.java | 35 ------
.../java/org/apach/asterix/cloud/s3/LSMS3Test.java | 6 +-
.../asterix/common/api/IApplicationContext.java | 7 ++
.../asterix/common/api/IPropertiesFactory.java | 3 +
.../asterix/common/config/AsterixProperties.java | 2 +-
.../asterix/common/config/CloudProperties.java | 120 +++++++++++++++++++++
.../asterix/common/config/PropertiesFactory.java | 5 +
29 files changed, 280 insertions(+), 560 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index 3fad2d19d0..0b8024bd96 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -37,6 +37,7 @@ import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
@@ -99,6 +100,7 @@ public class CcApplicationContext implements ICcApplicationContext {
private ExtensionProperties extensionProperties;
private MessagingProperties messagingProperties;
private NodeProperties nodeProperties;
+ private final CloudProperties cloudProperties;
private Supplier<IMetadataBootstrap> metadataBootstrapSupplier;
private volatile HyracksConnection hcc;
private volatile ResultSet resultSet;
@@ -140,6 +142,7 @@ public class CcApplicationContext implements ICcApplicationContext {
activeProperties = new ActiveProperties(propertiesAccessor);
extensionProperties = new ExtensionProperties(propertiesAccessor);
replicationProperties = new ReplicationProperties(propertiesAccessor);
+ cloudProperties = new CloudProperties(propertiesAccessor);
this.ftStrategy = ftStrategy;
this.buildProperties = new BuildProperties(propertiesAccessor);
this.messagingProperties = new MessagingProperties(propertiesAccessor);
@@ -373,4 +376,9 @@ public class CcApplicationContext implements ICcApplicationContext {
public boolean isCloudDeployment() {
return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
}
+
+ @Override
+ public CloudProperties getCloudProperties() {
+ return cloudProperties;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 7f0b529480..06115450d6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -42,6 +42,7 @@ import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IReceptionistFactory;
import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.MessagingProperties;
@@ -164,6 +165,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
private final ICacheManager cacheManager;
private IConfigValidator configValidator;
private IDiskWriteRateLimiterProvider diskWriteRateLimiterProvider;
+ private final CloudProperties cloudProperties;
public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager,
IPropertiesFactory propertiesFactory) {
@@ -178,6 +180,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
replicationProperties = propertiesFactory.newReplicationProperties();
messagingProperties = propertiesFactory.newMessagingProperties();
nodeProperties = propertiesFactory.newNodeProperties();
+ cloudProperties = propertiesFactory.newCloudProperties();
ncExtensionManager = extensionManager;
componentProvider = new StorageComponentProvider();
resourceIdFactory = new GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
@@ -191,7 +194,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
boolean initialRun) throws IOException {
ioManager = getServiceContext().getIoManager();
if (isCloudDeployment()) {
- persistenceIOManager = new CloudIOManager((IOManager) ioManager);
+ persistenceIOManager = new CloudIOManager((IOManager) ioManager, cloudProperties);
} else {
persistenceIOManager = ioManager;
}
@@ -661,4 +664,9 @@ public class NCAppRuntimeContext implements INcApplicationContext {
public boolean isCloudDeployment() {
return ncServiceContext.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
}
+
+ @Override
+ public CloudProperties getCloudProperties() {
+ return cloudProperties;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
deleted file mode 100644
index be9bd544ea..0000000000
--- a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf
+++ /dev/null
@@ -1,68 +0,0 @@
-; Licensed to the Apache Software Foundation (ASF) under one
-; or more contributor license agreements. See the NOTICE file
-; distributed with this work for additional information
-; regarding copyright ownership. The ASF licenses this file
-; to you under the Apache License, Version 2.0 (the
-; "License"); you may not use this file except in compliance
-; with the License. You may obtain a copy of the License at
-;
-; http://www.apache.org/licenses/LICENSE-2.0
-;
-; Unless required by applicable law or agreed to in writing,
-; software distributed under the License is distributed on an
-; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-; KIND, either express or implied. See the License for the
-; specific language governing permissions and limitations
-; under the License.
-
-[nc/asterix_nc1]
-txn.log.dir=target/tmp/asterix_nc1/txnlog
-core.dump.dir=target/tmp/asterix_nc1/coredump
-iodevices=asterix_nc1/iodevice1
-iodevices=asterix_nc1/iodevice2
-nc.api.port=19004
-#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
-
-[nc/asterix_nc2]
-ncservice.port=9091
-txn.log.dir=target/tmp/asterix_nc2/txnlog
-core.dump.dir=target/tmp/asterix_nc2/coredump
-iodevices=asterix_nc2/iodevice1
-iodevices=asterix_nc2/iodevice2
-nc.api.port=19005
-#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
-
-[nc]
-address=127.0.0.1
-command=asterixnc
-app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
-jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
-storage.buffercache.size=128MB
-storage.memorycomponent.globalbudget=512MB
-storage.io.scheduler=greedy
-storage.filtered.memorycomponent.max.size=16MB
-
-[cc]
-address = 127.0.0.1
-app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
-heartbeat.period=2000
-heartbeat.max.misses=25
-
-[common]
-log.dir = logs/
-log.level = INFO
-compiler.cbo=false
-compiler.cbotest=true
-compiler.queryplanshape=zigzag
-compiler.framesize=32KB
-compiler.sortmemory=320KB
-compiler.groupmemory=160KB
-compiler.joinmemory=256KB
-compiler.textsearchmemory=160KB
-compiler.windowmemory=192KB
-compiler.sort.parallel=false
-compiler.internal.sanitycheck=true
-messaging.frame.size=4096
-messaging.frame.count=512
-cloud.deployment=true
-storage.buffercache.pagesize=32KB
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
index 293a58041a..352dc067a4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
@@ -20,7 +20,6 @@ package org.apache.asterix.api.common;
import java.net.URI;
-import org.apache.asterix.cloud.storage.MockCloudStorageConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -36,15 +35,19 @@ public class CloudUtils {
private static final Logger LOGGER = LogManager.getLogger();
private static final int MOCK_SERVER_PORT = 8001;
- private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
- private static final String CLOUD_STORAGE_BUCKET = MockCloudStorageConfiguration.INSTANCE.getContainer();
- private static final String MOCK_SERVER_REGION = "us-west-2";
+ public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+ public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container";
+ public static final String MOCK_SERVER_REGION = "us-west-2";
private static S3Mock s3MockServer;
private CloudUtils() {
throw new AssertionError("Do not instantiate");
}
+ public static void main(String[] args) {
+ startS3CloudEnvironment();
+ }
+
public static void startS3CloudEnvironment() {
// Starting S3 mock server to be used instead of real S3 server
LOGGER.info("Starting S3 mock server");
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
index aaa2f38b90..505e239c85 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
@@ -45,7 +45,7 @@ public class CloudStorageTest {
private static final TestExecutor testExecutor = new TestExecutor();
private static final String SUITE_TESTS = "testsuite_cloud_storage.xml";
private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
- private static final String CONFIG_FILE_NAME = "src/main/resources/cc-cloud-storage.conf";
+ private static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage.conf";
public CloudStorageTest(TestCaseContext tcCtx) {
this.tcCtx = tcCtx;
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
index c3f9591e1e..6ffa3c06a3 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
@@ -61,4 +61,9 @@ compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
cloud.deployment=true
-storage.buffercache.pagesize=32KB
\ No newline at end of file
+storage.buffercache.pagesize=32KB
+cloud.storage.scheme=s3
+cloud.storage.bucket=cloud-storage-container
+cloud.storage.region=us-west-2
+cloud.storage.endpoint=http://127.0.0.1:8001
+cloud.storage.anonymous.auth=true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index e4eaa0c78f..74a17e0e23 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -10,6 +10,12 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.storage.anonymous.auth" : false,
+ "cloud.storage.bucket" : "",
+ "cloud.storage.endpoint" : "",
+ "cloud.storage.prefix" : "",
+ "cloud.storage.region" : "",
+ "cloud.storage.scheme" : "",
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 363be4c355..a61c0dd8aa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -10,6 +10,12 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.storage.anonymous.auth" : false,
+ "cloud.storage.bucket" : "",
+ "cloud.storage.endpoint" : "",
+ "cloud.storage.prefix" : "",
+ "cloud.storage.region" : "",
+ "cloud.storage.scheme" : "",
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index da5706f40c..fa3f5fa92c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -10,6 +10,12 @@
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
"cloud.deployment" : false,
+ "cloud.storage.anonymous.auth" : false,
+ "cloud.storage.bucket" : "",
+ "cloud.storage.endpoint" : "",
+ "cloud.storage.prefix" : "",
+ "cloud.storage.region" : "",
+ "cloud.storage.scheme" : "",
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index 799da68333..b508b6c930 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -36,12 +36,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.asterix.cloud.clients.CloudClientProvider;
-import org.apache.asterix.cloud.clients.CloudClientProvider.ClientType;
import org.apache.asterix.cloud.clients.ICloudClient;
-import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
-import org.apache.asterix.cloud.storage.CloudStorageConfigurationProvider;
-import org.apache.asterix.cloud.storage.ICloudStorageConfiguration;
-import org.apache.asterix.cloud.storage.ICloudStorageConfiguration.ConfigurationType;
+import org.apache.asterix.common.config.CloudProperties;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -62,19 +58,17 @@ public class CloudIOManager extends IOManager {
private IOManager localIoManager;
private CloudIOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism,
- int queueSize) throws HyracksDataException {
+ int queueSize, CloudProperties cloudProperties) throws HyracksDataException {
super(devices, deviceComputer, ioParallelism, queueSize);
- ICloudStorageConfiguration cloudStorageConfiguration =
- CloudStorageConfigurationProvider.INSTANCE.getConfiguration(ConfigurationType.FILE);
- this.bucket = cloudStorageConfiguration.getContainer();
- cloudClient = CloudClientProvider.getClient(ClientType.S3, CredentialsType.FILE);
+ this.bucket = cloudProperties.getStorageBucket();
+ cloudClient = CloudClientProvider.getClient(cloudProperties);
int numOfThreads = getIODevices().size() * getIoParallelism();
writeBufferProvider = new WriteBufferProvider(numOfThreads);
}
- public CloudIOManager(IOManager ioManager) throws HyracksDataException {
+ public CloudIOManager(IOManager ioManager, CloudProperties cloudProperties) throws HyracksDataException {
this(ioManager.getIoDevices(), ioManager.getDeviceComputer(), ioManager.getIoParallelism(),
- ioManager.getQueueSize());
+ ioManager.getQueueSize(), cloudProperties);
this.localIoManager = ioManager;
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index d808ea5468..21450c4d33 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -18,33 +18,22 @@
*/
package org.apache.asterix.cloud.clients;
-import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
+import org.apache.asterix.cloud.clients.aws.s3.S3ClientConfig;
import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
-import org.apache.asterix.cloud.clients.aws.s3.credentials.S3CredentialsProvider;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.asterix.common.config.CloudProperties;
public class CloudClientProvider {
- public enum ClientType {
- NO_OP,
- S3,
- AZURE_BLOB,
- GOOGLE_CLOUD_STORAGE
- }
-
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
}
- public static ICloudClient getClient(ClientType clientType, CredentialsType credentialsType)
- throws HyracksDataException {
- switch (clientType) {
- case S3:
- IS3Credentials credentials = S3CredentialsProvider.INSTANCE.getCredentials(credentialsType);
- return new S3CloudClient(credentials);
- default:
- throw HyracksDataException.create(new IllegalArgumentException("Unknown cloud client type"));
+ public static ICloudClient getClient(CloudProperties cloudProperties) {
+ String storageScheme = cloudProperties.getStorageScheme();
+ if ("s3".equalsIgnoreCase(storageScheme)) {
+ S3ClientConfig config = S3ClientConfig.of(cloudProperties);
+ return new S3CloudClient(config);
}
+ throw new IllegalStateException("unsupported cloud storage scheme: " + storageScheme);
}
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
deleted file mode 100644
index 1bb94b2666..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClientCredentialsProvider.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface ICloudClientCredentialsProvider {
- enum CredentialsType {
- FILE,
- MOCK
- }
-
- ICredentials getCredentials(CredentialsType type) throws HyracksDataException;
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
deleted file mode 100644
index abf5284caf..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICredentials.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients;
-
-public interface ICredentials {
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
new file mode 100644
index 0000000000..281fe7707b
--- /dev/null
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients.aws.s3;
+
+import org.apache.asterix.common.config.CloudProperties;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+
+public class S3ClientConfig {
+
+ private final String region;
+ private final String endpoint;
+ private final String prefix;
+ private final boolean anonymousAuth;
+
+ public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth) {
+ this.region = region;
+ this.endpoint = endpoint;
+ this.prefix = prefix;
+ this.anonymousAuth = anonymousAuth;
+ }
+
+ public static S3ClientConfig of(CloudProperties cloudProperties) {
+ return new S3ClientConfig(cloudProperties.getStorageRegion(), cloudProperties.getStorageEndpoint(),
+ cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth());
+ }
+
+ public String getRegion() {
+ return region;
+ }
+
+ public String getEndpoint() {
+ return endpoint;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public AwsCredentialsProvider createCredentialsProvider() {
+ return anonymousAuth ? AnonymousCredentialsProvider.create() : DefaultCredentialsProvider.create();
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 0b78682e4f..97b164fbcf 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -26,6 +26,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -39,7 +40,6 @@ import java.util.concurrent.ExecutionException;
import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.asterix.cloud.clients.ICloudClient;
-import org.apache.asterix.cloud.clients.aws.s3.credentials.IS3Credentials;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -47,9 +47,6 @@ import org.apache.hyracks.api.util.IoUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
@@ -76,32 +73,30 @@ public class S3CloudClient implements ICloudClient {
private static final Logger LOGGER = LogManager.getLogger();
- private final IS3Credentials credentials;
+ private final S3ClientConfig config;
private final S3Client s3Client;
private S3TransferManager s3TransferManager;
// TODO(htowaileb): Temporary variables, can we get this from the used instance?
private static final double MAX_HOST_BANDWIDTH = 10.0; // in Gbps
- public S3CloudClient(IS3Credentials credentials) throws HyracksDataException {
- this.credentials = credentials;
+ public S3CloudClient(S3ClientConfig config) {
+ this.config = config;
s3Client = buildClient();
}
- private S3Client buildClient() throws HyracksDataException {
- AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider
- .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey()));
+ private S3Client buildClient() {
S3ClientBuilder builder = S3Client.builder();
- builder.credentialsProvider(credentialsProvider);
- builder.region(Region.of(credentials.getRegion()));
-
- if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
+ builder.credentialsProvider(config.createCredentialsProvider());
+ builder.region(Region.of(config.getRegion()));
+ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
+ URI uri;
try {
- URI uri = new URI(credentials.getEndpoint());
- builder.endpointOverride(uri);
- } catch (Exception ex) {
- throw HyracksDataException.create(ex);
+ uri = new URI(config.getEndpoint());
+ } catch (URISyntaxException ex) {
+ throw new IllegalArgumentException(ex);
}
+ builder.endpointOverride(uri);
}
return builder.build();
}
@@ -321,14 +316,13 @@ public class S3CloudClient implements ICloudClient {
}
S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder();
- builder.credentialsProvider(StaticCredentialsProvider
- .create(AwsBasicCredentials.create(credentials.getAccessKeyId(), credentials.getSecretAccessKey())));
- builder.region(Region.of(credentials.getRegion()));
+ builder.credentialsProvider(config.createCredentialsProvider());
+ builder.region(Region.of(config.getRegion()));
builder.targetThroughputInGbps(MAX_HOST_BANDWIDTH);
builder.minimumPartSizeInBytes((long) 8 * 1024 * 1024);
- if (credentials.getEndpoint() != null && !credentials.getEndpoint().isEmpty()) {
- builder.endpointOverride(URI.create(credentials.getEndpoint()));
+ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
+ builder.endpointOverride(URI.create(config.getEndpoint()));
}
S3AsyncClient client = builder.build();
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
deleted file mode 100644
index 342e1a15ce..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/FileCredentials.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients.aws.s3.credentials;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class FileCredentials implements IS3Credentials {
-
- private final String accessKeyId;
- private final String secretAccessKey;
- private final String region;
- private String endpoint;
-
- // TODO(htowaileb): change the credential file to be json object instead of reading per line
- public FileCredentials(File file) throws HyracksDataException {
- if (!file.exists()) {
- throw new IllegalStateException("No cloud configuration file found");
- }
-
- try {
- List<String> lines = FileUtils.readLines(file, "UTF-8");
- this.accessKeyId = lines.get(1);
- this.secretAccessKey = lines.get(2);
- this.region = lines.get(3);
-
- if (lines.size() > 4) {
- this.endpoint = lines.get(4);
- }
- } catch (IOException ex) {
- throw HyracksDataException.create(ex);
- }
- }
-
- @Override
- public String getAccessKeyId() {
- return accessKeyId;
- }
-
- @Override
- public String getSecretAccessKey() {
- return secretAccessKey;
- }
-
- @Override
- public String getRegion() {
- return region;
- }
-
- @Override
- public String getEndpoint() {
- return endpoint;
- }
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
deleted file mode 100644
index 32da3da176..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/IS3Credentials.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients.aws.s3.credentials;
-
-import org.apache.asterix.cloud.clients.ICredentials;
-
-public interface IS3Credentials extends ICredentials {
-
- /**
- * @return access key id
- */
- String getAccessKeyId();
-
- /**
- * @return secret access key
- */
- String getSecretAccessKey();
-
- /**
- * @return region
- */
- String getRegion();
-
- /**
- * @return endpoint
- */
- String getEndpoint();
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
deleted file mode 100644
index e605ea1058..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3CredentialsProvider.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients.aws.s3.credentials;
-
-import java.io.File;
-
-import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class S3CredentialsProvider implements ICloudClientCredentialsProvider {
-
- public static S3CredentialsProvider INSTANCE = new S3CredentialsProvider();
-
- private S3CredentialsProvider() {
- }
-
- @Override
- public IS3Credentials getCredentials(CredentialsType type) throws HyracksDataException {
- switch (type) {
- case FILE:
- File file = new File("/etc/s3");
- if (file.exists()) {
- return new FileCredentials(file);
- }
- default:
- return S3MockCredentials.INSTANCE;
- }
- }
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
deleted file mode 100644
index 56bb08a210..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/credentials/S3MockCredentials.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.clients.aws.s3.credentials;
-
-public class S3MockCredentials implements IS3Credentials {
-
- private static final String ACCESS_KEY_ID = "dummyAccessKeyId";
- private static final String SECRET_ACCESS_KEY = "dummySecretAccessKey";
- private static final String REGION = "us-west-2";
- private static final String ENDPOINT = "http://127.0.0.1:8001";
-
- public static final S3MockCredentials INSTANCE = new S3MockCredentials();
-
- private S3MockCredentials() {
- }
-
- @Override
- public String getAccessKeyId() {
- return ACCESS_KEY_ID;
- }
-
- @Override
- public String getSecretAccessKey() {
- return SECRET_ACCESS_KEY;
- }
-
- @Override
- public String getRegion() {
- return REGION;
- }
-
- @Override
- public String getEndpoint() {
- return ENDPOINT;
- }
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
deleted file mode 100644
index 7a94f9825d..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/CloudStorageConfigurationProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.storage;
-
-import java.io.File;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class CloudStorageConfigurationProvider {
-
- public static CloudStorageConfigurationProvider INSTANCE = new CloudStorageConfigurationProvider();
-
- private CloudStorageConfigurationProvider() {
- }
-
- public ICloudStorageConfiguration getConfiguration(ICloudStorageConfiguration.ConfigurationType type)
- throws HyracksDataException {
- switch (type) {
- case FILE:
- File file = new File("/etc/storage");
- if (file.exists()) {
- return new FileCloudStorageConfiguration(file);
- }
- default:
- return MockCloudStorageConfiguration.INSTANCE;
- }
- }
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
deleted file mode 100644
index ef75bba8e8..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.storage;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class FileCloudStorageConfiguration implements ICloudStorageConfiguration {
-
- private final String containerName;
-
- public FileCloudStorageConfiguration(File file) throws HyracksDataException {
- if (!file.exists()) {
- throw new IllegalStateException("No cloud configuration file found");
- }
-
- try {
- List<String> lines = FileUtils.readLines(file, "UTF-8");
- this.containerName = lines.get(0);
- } catch (IOException ex) {
- throw HyracksDataException.create(ex);
- }
- }
-
- @Override
- public String getContainer() {
- return containerName;
- }
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
deleted file mode 100644
index 356b2ef06d..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.storage;
-
-public interface ICloudStorageConfiguration {
-
- enum ConfigurationType {
- FILE,
- MOCK
- }
-
- /**
- * @return returns the container name used for the storage
- */
- String getContainer();
-}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
deleted file mode 100644
index 1483739c4b..0000000000
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.cloud.storage;
-
-public class MockCloudStorageConfiguration implements ICloudStorageConfiguration {
-
- private static final String CONTAINER_NAME = "cloud-storage-container";
-
- public static MockCloudStorageConfiguration INSTANCE = new MockCloudStorageConfiguration();
-
- private MockCloudStorageConfiguration() {
-
- }
-
- @Override
- public String getContainer() {
- return CONTAINER_NAME;
- }
-}
diff --git a/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
index 2424cede93..91b47e227f 100644
--- a/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apach/asterix/cloud/s3/LSMS3Test.java
@@ -21,8 +21,8 @@ package org.apach.asterix.cloud.s3;
import java.net.URI;
import org.apach.asterix.cloud.LSMTest;
+import org.apache.asterix.cloud.clients.aws.s3.S3ClientConfig;
import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
-import org.apache.asterix.cloud.clients.aws.s3.credentials.S3MockCredentials;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -64,8 +64,8 @@ public class LSMS3Test extends LSMTest {
cleanup();
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
LOGGER.info("Client created successfully");
-
- CLOUD_CLIENT = new S3CloudClient(S3MockCredentials.INSTANCE);
+ S3ClientConfig config = new S3ClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", true);
+ CLOUD_CLIENT = new S3CloudClient(config);
}
private static void cleanup() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index def3634139..812e2e55f1 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -20,6 +20,7 @@ package org.apache.asterix.common.api;
import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.MessagingProperties;
@@ -95,4 +96,10 @@ public interface IApplicationContext {
* @return true if running in cloud deployment, false otherwise.
*/
boolean isCloudDeployment();
+
+ /**
+ * Gets the cloud properties
+ * @return the cloud properties
+ */
+ CloudProperties getCloudProperties();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java
index 7857f18fc3..1d3f14eaf0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java
@@ -20,6 +20,7 @@ package org.apache.asterix.common.api;
import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.MessagingProperties;
@@ -100,4 +101,6 @@ public interface IPropertiesFactory {
* @return new node properties
*/
NodeProperties newNodeProperties();
+
+ CloudProperties newCloudProperties();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
index d192b495e0..2b91ded3f5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
@@ -37,7 +37,7 @@ public class AsterixProperties {
configManager.register(NodeProperties.Option.class, CompilerProperties.Option.class,
MetadataProperties.Option.class, ExternalProperties.Option.class, ActiveProperties.Option.class,
MessagingProperties.Option.class, ReplicationProperties.Option.class, StorageProperties.Option.class,
- TransactionProperties.Option.class);
+ TransactionProperties.Option.class, CloudProperties.Option.class);
// we need to process the old-style asterix config before we apply defaults!
configManager.addConfigurator(IConfigManager.ConfiguratorMetric.APPLY_DEFAULTS.metric() - 1, () -> {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
new file mode 100644
index 0000000000..a48016743c
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
+import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
+
+import org.apache.hyracks.api.config.IOption;
+import org.apache.hyracks.api.config.IOptionType;
+import org.apache.hyracks.api.config.Section;
+
+public class CloudProperties extends AbstractProperties {
+
+ public CloudProperties(PropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public enum Option implements IOption {
+ CLOUD_STORAGE_SCHEME(STRING, ""),
+ CLOUD_STORAGE_BUCKET(STRING, ""),
+ CLOUD_STORAGE_PREFIX(STRING, ""),
+ CLOUD_STORAGE_REGION(STRING, ""),
+ CLOUD_STORAGE_ENDPOINT(STRING, ""),
+ CLOUD_STORAGE_ANONYMOUS_AUTH(BOOLEAN, false);
+
+ private final IOptionType interpreter;
+ private final Object defaultValue;
+
+ <T> Option(IOptionType<T> interpreter, T defaultValue) {
+ this.interpreter = interpreter;
+ this.defaultValue = defaultValue;
+ }
+
+ @Override
+ public Section section() {
+ switch (this) {
+ case CLOUD_STORAGE_SCHEME:
+ case CLOUD_STORAGE_BUCKET:
+ case CLOUD_STORAGE_PREFIX:
+ case CLOUD_STORAGE_REGION:
+ case CLOUD_STORAGE_ENDPOINT:
+ case CLOUD_STORAGE_ANONYMOUS_AUTH:
+ return Section.COMMON;
+ default:
+ return Section.NC;
+ }
+ }
+
+ @Override
+ public String description() {
+ switch (this) {
+ case CLOUD_STORAGE_SCHEME:
+ return "The cloud storage scheme e.g. (s3)";
+ case CLOUD_STORAGE_BUCKET:
+ return "The cloud storage bucket name";
+ case CLOUD_STORAGE_PREFIX:
+ return "The cloud storage path prefix";
+ case CLOUD_STORAGE_REGION:
+ return "The cloud storage region";
+ case CLOUD_STORAGE_ENDPOINT:
+ return "The cloud storage endpoint";
+ case CLOUD_STORAGE_ANONYMOUS_AUTH:
+ return "Indicates whether or not anonymous auth should be used for the cloud storage";
+ default:
+ throw new IllegalStateException("NYI: " + this);
+ }
+ }
+
+ @Override
+ public IOptionType type() {
+ return interpreter;
+ }
+
+ @Override
+ public Object defaultValue() {
+ return defaultValue;
+ }
+
+ }
+
+ public String getStorageScheme() {
+ return accessor.getString(Option.CLOUD_STORAGE_SCHEME);
+ }
+
+ public String getStorageBucket() {
+ return accessor.getString(Option.CLOUD_STORAGE_BUCKET);
+ }
+
+ public String getStoragePrefix() {
+ return accessor.getString(Option.CLOUD_STORAGE_PREFIX);
+ }
+
+ public String getStorageEndpoint() {
+ return accessor.getString(Option.CLOUD_STORAGE_ENDPOINT);
+ }
+
+ public String getStorageRegion() {
+ return accessor.getString(Option.CLOUD_STORAGE_REGION);
+ }
+
+ public boolean isStorageAnonymousAuth() {
+ return accessor.getBoolean(Option.CLOUD_STORAGE_ANONYMOUS_AUTH);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java
index 8f75397c62..87c853da46 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java
@@ -77,4 +77,9 @@ public class PropertiesFactory implements IPropertiesFactory {
public NodeProperties newNodeProperties() {
return new NodeProperties(propertiesAccessor);
}
+
+ @Override
+ public CloudProperties newCloudProperties() {
+ return new CloudProperties(propertiesAccessor);
+ }
}