You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/02/26 17:03:30 UTC
[airavata-mft] branch master updated: Connection pooling for swift
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new a8fb679 Connection pooling for swift
a8fb679 is described below
commit a8fb6790da5aeb463494fc106781c705408b97ca
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Sun Feb 26 12:03:01 2023 -0500
Connection pooling for swift
---
.../transport/swift/SwiftIncomingConnector.java | 14 ++---
.../transport/swift/SwiftMetadataCollector.java | 10 ++--
.../transport/swift/SwiftOutgoingConnector.java | 13 ++--
.../airavata/mft/transport/swift/SwiftUtil.java | 69 ++++++++++++++++++----
4 files changed, 71 insertions(+), 35 deletions(-)
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
index 5f9a415..4c9ab88 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
@@ -20,13 +20,8 @@ package org.apache.airavata.mft.transport.swift;
import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.IncomingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
-import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
-import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
-import org.jclouds.ContextBuilder;
import org.jclouds.http.options.GetOptions;
-import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
-import org.jclouds.openstack.keystone.config.KeystoneProperties;
import org.jclouds.openstack.swift.v1.SwiftApi;
import org.jclouds.openstack.swift.v1.domain.SwiftObject;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
@@ -36,7 +31,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.InputStream;
import java.nio.file.StandardCopyOption;
-import java.util.Properties;
public class SwiftIncomingConnector implements IncomingChunkedConnector {
@@ -52,15 +46,15 @@ public class SwiftIncomingConnector implements IncomingChunkedConnector {
SwiftStorage swiftStorage = cc.getStorage().getSwift();
this.resourcePath = cc.getResourcePath();
SwiftSecret swiftSecret = cc.getSecret().getSwift();
- swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage);
+ swiftApi = SwiftUtil.getInstance().leaseSwiftApi(swiftSecret);
objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
}
@Override
public void complete() throws Exception {
- if (swiftApi != null) {
- swiftApi.close();
- }
+ //if (swiftApi != null) {
+ // swiftApi.close();
+ //}
}
@Override
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
index ad4fa85..9073133 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java
@@ -28,8 +28,6 @@ import org.jclouds.openstack.swift.v1.domain.ObjectList;
import org.jclouds.openstack.swift.v1.domain.SwiftObject;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
-import java.util.Properties;
-
public class SwiftMetadataCollector implements MetadataCollector {
boolean initialized = false;
private SwiftStorage swiftStorage;
@@ -52,7 +50,7 @@ public class SwiftMetadataCollector implements MetadataCollector {
public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception {
checkInitialized();
- SwiftApi swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage);
+ SwiftApi swiftApi = SwiftUtil.getInstance().leaseSwiftApi(swiftSecret);
try {
ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder();
@@ -105,7 +103,7 @@ public class SwiftMetadataCollector implements MetadataCollector {
}
return resourceBuilder.build();
} finally{
- swiftApi.close();
+ //swiftApi.close();
}
}
@@ -113,7 +111,7 @@ public class SwiftMetadataCollector implements MetadataCollector {
public Boolean isAvailable(String resourcePath) throws Exception {
checkInitialized();
- SwiftApi swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage);
+ SwiftApi swiftApi = SwiftUtil.getInstance().leaseSwiftApi(swiftSecret);
try {
ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
@@ -122,7 +120,7 @@ public class SwiftMetadataCollector implements MetadataCollector {
return swiftObject != null;
} finally {
- swiftApi.close();
+ //swiftApi.close();
}
}
}
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
index 31132d5..431f5a8 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
@@ -20,13 +20,8 @@ package org.apache.airavata.mft.transport.swift;
import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.OutgoingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
-import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
-import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
-import org.jclouds.ContextBuilder;
import org.jclouds.io.payloads.InputStreamPayload;
-import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
-import org.jclouds.openstack.keystone.config.KeystoneProperties;
import org.jclouds.openstack.swift.v1.SwiftApi;
import org.jclouds.openstack.swift.v1.domain.Segment;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
@@ -60,7 +55,7 @@ public class SwiftOutgoingConnector implements OutgoingChunkedConnector {
this.resourcePath = cc.getResourcePath();
SwiftSecret swiftSecret = cc.getSecret().getSwift();
- swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage);
+ swiftApi = SwiftUtil.getInstance().leaseSwiftApi(swiftSecret);
objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
staticLargeObjectApi = swiftApi.getStaticLargeObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer());
}
@@ -76,9 +71,9 @@ public class SwiftOutgoingConnector implements OutgoingChunkedConnector {
String etag = staticLargeObjectApi.replaceManifest(resourcePath,
segments, new HashMap<>());
- if (swiftApi != null) {
- swiftApi.close();
- }
+ //if (swiftApi != null) {
+ // swiftApi.close();
+ //}
}
@Override
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java
index 4145854..66c24ac 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java
@@ -17,31 +17,77 @@
package org.apache.airavata.mft.transport.swift;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftV2AuthSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftV3AuthSecret;
-import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
import org.jclouds.ContextBuilder;
import org.jclouds.openstack.keystone.config.KeystoneProperties;
import org.jclouds.openstack.swift.v1.SwiftApi;
// https://jclouds.apache.org/guides/openstack/
public class SwiftUtil {
- public static SwiftApi createSwiftApi(SwiftSecret swiftSecret, SwiftStorage swiftStorage) throws Exception {
- String provider = "openstack-swift";
- Properties overrides = new Properties();
- switch (swiftSecret.getSecretCase()) {
+ private ThreadLocal<Map<String, SwiftApi>> swiftApiCache = ThreadLocal.withInitial(() -> {
+ Map<String, SwiftApi> map = new HashMap<>();
+ return map;
+ });
+
+ private static SwiftUtil instance;
+
+ private SwiftUtil(){}
+
+ public static synchronized SwiftUtil getInstance() {
+ if (instance == null) {
+ synchronized (SwiftUtil.class) {
+ if (instance == null) {
+ instance = new SwiftUtil();
+ }
+ }
+ }
+ return instance;
+ }
+
+ public void releaseSwiftApi(SwiftSecret swiftSecret) {
+
+ }
+
+ private String getSecretKey(SwiftSecret swiftSecret) {
+ switch (swiftSecret.getSecretCase()) {
+ case V2AUTHSECRET:
+ SwiftV2AuthSecret v2AuthSecret = swiftSecret.getV2AuthSecret();
+ return v2AuthSecret.getTenant() + v2AuthSecret.getUserName() + v2AuthSecret.getPassword();
+ case V3AUTHSECRET:
+ SwiftV3AuthSecret v3AuthSecret = swiftSecret.getV3AuthSecret();
+ return v3AuthSecret.getTenantName() + v3AuthSecret.getProjectDomainName()
+ + v3AuthSecret.getUserDomainName() + v3AuthSecret.getUserName() + v3AuthSecret.getPassword();
+ }
+ return null;
+ }
+ public SwiftApi leaseSwiftApi(SwiftSecret swiftSecret) throws Exception {
+
+ String secretKey = getSecretKey(swiftSecret);
+
+ if (swiftApiCache.get().containsKey(secretKey)) {
+ return swiftApiCache.get().get(secretKey);
+ }
+
+ SwiftApi swiftApi;
+
+ String provider = "openstack-swift";
+ Properties overrides = new Properties();
+ switch (swiftSecret.getSecretCase()) {
case V2AUTHSECRET:
SwiftV2AuthSecret v2AuthSecret = swiftSecret.getV2AuthSecret();
overrides.put(KeystoneProperties.KEYSTONE_VERSION, "2");
- return ContextBuilder.newBuilder(provider)
+ swiftApi = ContextBuilder.newBuilder(provider)
.endpoint(swiftSecret.getEndpoint())
.credentials(v2AuthSecret.getTenant() + ":" + v2AuthSecret.getUserName(),
v2AuthSecret.getPassword())
.overrides(overrides)
.buildApi(SwiftApi.class);
-
+ break;
case V3AUTHSECRET:
SwiftV3AuthSecret v3AuthSecret = swiftSecret.getV3AuthSecret();
@@ -54,15 +100,18 @@ public class SwiftUtil {
overrides.put(KeystoneProperties.PROJECT_DOMAIN_NAME, v3AuthSecret.getProjectDomainName());
}
- return ContextBuilder.newBuilder(provider)
+ swiftApi = ContextBuilder.newBuilder(provider)
.endpoint(swiftSecret.getEndpoint())
.credentials(v3AuthSecret.getUserDomainName() + ":" + v3AuthSecret.getUserName(),
v3AuthSecret.getPassword())
.overrides(overrides)
.buildApi(SwiftApi.class);
-
+ break;
default:
throw new Exception("No v2 or v3 auth secret set");
- }
+ }
+
+ swiftApiCache.get().put(secretKey, swiftApi);
+ return swiftApi;
}
}