You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/08/22 16:42:16 UTC
[airavata-mft] 10/22: Adding NIO based file transferring initial
impl
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit c7b1eb4e1f325faa401e074927c8529277ef216a
Author: isururanawaka <ir...@gmail.com>
AuthorDate: Thu Aug 15 13:24:40 2019 +0530
Adding NIO based file transferring initial impl
---
core/pom.xml | 12 +++
.../airavata/mft/core/api/CompletionCallback.java | 40 ++++++++++
.../apache/airavata/mft/core/api/Connector.java | 51 ++++++++++++
.../airavata/mft/core/api/ConnectorChannel.java | 58 ++++++++++++++
.../org/apache/airavata/mft/core/api/Mediator.java | 39 +++++++++
.../airavata/mft/core/api/SinkConnector.java | 43 ++++++++++
.../airavata/mft/core/api/SourceConnector.java | 37 +++++++++
.../mft/core/bufferedImpl/AbstractConnector.java | 75 +++++++++++++++++
.../mft/core/bufferedImpl/ChannelUtils.java | 87 ++++++++++++++++++++
.../mft/core/bufferedImpl/ConnectorConfig.java | 39 +++++++++
.../mft/core/bufferedImpl/ConnectorException.java | 37 +++++++++
.../airavata/mft/core/bufferedImpl/Constants.java | 30 +++++++
.../airavata/mft/core/bufferedImpl/InChannel.java | 72 +++++++++++++++++
.../airavata/mft/core/bufferedImpl/OutChannel.java | 69 ++++++++++++++++
.../mft/core/bufferedImpl/PassthroughMediator.java | 83 +++++++++++++++++++
.../org/apache/airavata/mft/transport/s3/Main.java | 90 +++++++++++++++++++++
.../airavata/mft/transport/s3/S3Constants.java | 35 ++++++++
.../airavata/mft/transport/s3/S3SinkConnector.java | 93 ++++++++++++++++++++++
.../mft/transport/s3/S3SourceConnector.java | 65 +++++++++++++++
19 files changed, 1055 insertions(+)
diff --git a/core/pom.xml b/core/pom.xml
index 1a4e07e..ef33495 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,6 +31,18 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>mft-core</artifactId>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>8</source>
+ <target>8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/CompletionCallback.java b/core/src/main/java/org/apache/airavata/mft/core/api/CompletionCallback.java
new file mode 100644
index 0000000..657b017
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/CompletionCallback.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+
+import com.sun.istack.internal.Nullable;
+
+/**
+ * This represents the callback of {@link Mediator}
+ * which indicates the completion state of mediation
+ */
+@FunctionalInterface
+public interface CompletionCallback {
+
+ /**
+ * Implementation of this should contain the logic
+ * for execution after mediation returns
+ * @param message
+ * @param error
+ */
+ void onComplete(String message, @Nullable Exception error);
+
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java b/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java
new file mode 100644
index 0000000..69be278
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+import org.apache.airavata.mft.core.bufferedImpl.ConnectorConfig;
+
+import java.nio.channels.Channel;
+import java.util.Properties;
+
+/**
+ * This represents a connection between external source or sink
+ */
+public interface Connector {
+
+ /**
+ * Initiates the connector object
+ * @param connectorConfig
+ * @return initation state whether success or not
+ */
+ boolean initiate(ConnectorConfig connectorConfig);
+
+ /**
+ * This returns a {@link ConnectorChannel}
+ * @return Channel
+ */
+ ConnectorChannel openChannel(Properties properties) throws Exception;
+
+ /**
+ * This is used to close the channel and release resources related to channel
+ * @param channel
+ * @throws Exception
+ */
+ void closeChannel(Channel channel) throws Exception;
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorChannel.java b/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorChannel.java
new file mode 100644
index 0000000..d6065b9
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/ConnectorChannel.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+import java.io.IOException;
+import java.nio.channels.Channel;
+
+/**
+ * An interface represents the underlying channel of a {@Link Connector}
+ */
+public interface ConnectorChannel {
+ /**
+ * get the channel of given connector
+ *
+ * @return
+ */
+ Channel getChannel();
+
+ /**
+ * close the channel if open else return
+ *
+ * @throws IOException
+ */
+ void closeChannel() throws IOException;
+
+ /**
+ * Save channel attribute to ConnectorChannel context
+ *
+ * @param key
+ * @param value
+ */
+ void addChannelAttribute(String key, Object value);
+
+ /**
+ * Get channel attribute from ConnectorChannel context
+ *
+ * @param key
+ * @return
+ */
+ Object getChannelAttribute(String key);
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/Mediator.java b/core/src/main/java/org/apache/airavata/mft/core/api/Mediator.java
new file mode 100644
index 0000000..e3ea081
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/Mediator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+
+/**
+ * This represent the interface for all mediators, which mediates
+ * data from source connectors and send them to sink connectors.
+ */
+public interface Mediator {
+
+ /**
+ * Mediates the content from source connector and writes the mediated content
+ * to destination connector
+ * WritableByteChannel
+ * @param src
+ * @param dst
+ * @param callback
+ */
+ void mediate(ConnectorChannel src, ConnectorChannel dst, CompletionCallback callback);
+
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/SinkConnector.java b/core/src/main/java/org/apache/airavata/mft/core/api/SinkConnector.java
new file mode 100644
index 0000000..4d2eb65
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/SinkConnector.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+import java.nio.channels.Channel;
+import java.util.Properties;
+
+/**
+ * This represents the output connector, where to write
+ * data from the application.
+ */
+public interface SinkConnector extends Connector{
+
+ /**
+ * provides the channel to write data to external location
+ * @return ConnectorChannel
+ */
+ ConnectorChannel openChannel(Properties properties) throws Exception;
+
+ /**
+ * Verify content upload of the given channel is completed
+ * @param channel
+ * @return true if succes else false
+ */
+ boolean verifyUpload(Channel channel);
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/SourceConnector.java b/core/src/main/java/org/apache/airavata/mft/core/api/SourceConnector.java
new file mode 100644
index 0000000..a6b1ac5
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/api/SourceConnector.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.api;
+
+import java.util.Properties;
+
+
+/**
+ * This represents the input connector, where to read data
+ * from the application
+ */
+public interface SourceConnector extends Connector{
+
+ /**
+ * provides the channel to read data from external location
+ * @return ConnectorChannel
+ */
+ ConnectorChannel openChannel(Properties properties);
+
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/AbstractConnector.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/AbstractConnector.java
new file mode 100644
index 0000000..2045356
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/AbstractConnector.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.core.api.SinkConnector;
+import org.apache.airavata.mft.core.api.SourceConnector;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.Channel;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Common class for {@link SinkConnector} and {@link SourceConnector} implementations
+ * This overrides closeChannel method and release all resources allocated to releasing channel
+ */
+public abstract class AbstractConnector implements Connector {
+ ConcurrentHashMap<Channel, Object> channelToStreamMap = new ConcurrentHashMap<>();
+
+ @Override
+ public void closeChannel(Channel channel) throws ConnectorException {
+ OutChannel outChannel;
+ InChannel inChannel;
+
+ try {
+ if (channelToStreamMap.get(channel) instanceof OutputStream) {
+ outChannel = (OutChannel) channelToStreamMap.get(channel);
+ if (outChannel != null) {
+ outChannel.closeChannel();
+ System.out.println("Closing output stream");
+ }
+
+ } else if (channelToStreamMap.get(channel) instanceof InputStream) {
+ inChannel = (InChannel) channelToStreamMap.get(channel);
+ if (inChannel != null) {
+ inChannel.closeChannel();
+ }
+ }
+
+ } catch (IOException e) {
+ throw new ConnectorException("Error occurred while closing stream", e);
+ }
+
+ }
+
+ public void cacheChannel(Channel channel, Object obj) {
+ channelToStreamMap.put(channel, obj);
+ }
+
+
+ public Object getConnectorChannel(Channel channel) {
+ return channelToStreamMap.get(channel);
+ }
+
+
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ChannelUtils.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ChannelUtils.java
new file mode 100644
index 0000000..1172142
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ChannelUtils.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * Contains the utility methods for Channel Operations
+ */
+public class ChannelUtils {
+
+ /**
+ * Tranfer data from readable byte channel to FileChannel using zero-copy
+ * @param byteChannel
+ * @param to
+ * @throws IOException
+ */
+ public static void transferFrom(ReadableByteChannel byteChannel, FileChannel to) throws IOException {
+ long count;
+ long total = 0;
+ while ((count = to.transferFrom(byteChannel, total, Constants.TRANSFER_MAX_SIZE)) > 0) {
+ total = +count;
+ }
+ }
+
+ /**
+ * Transfer data from FileChannel to writable byte channel using zero-copy
+ * @param to
+ * @param from
+ * @throws IOException
+ */
+ public static void transferTo(WritableByteChannel to, FileChannel from) throws IOException {
+ long count;
+ long total = 0;
+ while ((count = from.transferTo(total, Constants.TRANSFER_MAX_SIZE, to)) > 0) {
+ total = +count;
+ }
+ }
+
+ /**
+ * Copy data from readable byte channel to writable byte channel
+ * @param src
+ * @param dest
+ * @throws IOException
+ */
+ public static void copyData(ReadableByteChannel src, WritableByteChannel dest) throws IOException {
+ final ByteBuffer buffer = ByteBuffer.allocateDirect(Constants.BUFFER_SIZE);
+ int count = 0;
+ while ((count =src.read(buffer)) != -1) {
+ // prepare the buffer to be drained
+ buffer.flip();
+ // write to the channel, may block
+ dest.write(buffer);
+ // If partial transfer, shift remainder down
+ // If buffer is empty, same as doing clear()
+ buffer.compact();
+ }
+ // EOF will leave buffer in fill state
+ buffer.flip();
+ // make sure the buffer is fully drained.
+ while (buffer.hasRemaining()) {
+ dest.write(buffer);
+ }
+
+ }
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorConfig.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorConfig.java
new file mode 100644
index 0000000..240cd2a
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+
+import java.util.Properties;
+
+/**
+ * This class wraps all the content related to connector
+ */
+public class ConnectorConfig {
+
+ private Properties properties;
+
+ public ConnectorConfig(Properties properties) {
+ this.properties = properties;
+ }
+
+ public String getValue(String key) {
+ return properties.getProperty(key);
+ }
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorException.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorException.java
new file mode 100644
index 0000000..6fa1371
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/ConnectorException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+/**
+ * This class encapulate all the errors releated to Connector
+ * Configurations
+ */
+public class ConnectorException extends Exception {
+ private String msg;
+
+ public ConnectorException(String message, Throwable cause) {
+ super(message, cause);
+ this.msg = msg;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/Constants.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/Constants.java
new file mode 100644
index 0000000..1f5b46c
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/Constants.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+/**
+ * Contains all the constants used in core classes.
+ */
+public interface Constants {
+
+ long TRANSFER_MAX_SIZE = (1024 * 1024);
+ int BUFFER_SIZE = 8 * 1024;
+ String CONNECTOR = "CONNECTOR";
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/InChannel.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/InChannel.java
new file mode 100644
index 0000000..542f283
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/InChannel.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+import org.apache.airavata.mft.core.api.ConnectorChannel;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.HashMap;
+
+/**
+ * A class which represents the channel of a {@Link SinkConnector}
+ */
+public class InChannel implements ConnectorChannel {
+
+ private InputStream inputStream;
+
+ private ReadableByteChannel readableByteChannel;
+
+ private HashMap<String, Object> contextAttributeMap = new HashMap<>();
+
+ public InChannel(InputStream inputStream) {
+ this.inputStream = inputStream;
+ readableByteChannel = Channels.newChannel(inputStream);
+
+ }
+
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+
+ @Override
+ public Channel getChannel() {
+ return readableByteChannel;
+ }
+
+ @Override
+ public void closeChannel() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public void addChannelAttribute(String key, Object value) {
+ contextAttributeMap.put(key,value );
+ }
+
+ @Override
+ public Object getChannelAttribute(String key) {
+ return contextAttributeMap.get(key);
+ }
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/OutChannel.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/OutChannel.java
new file mode 100644
index 0000000..5d4e3c8
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/OutChannel.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+import org.apache.airavata.mft.core.api.ConnectorChannel;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.HashMap;
+
+/**
+ * A class which represents the channel in {@Link SinkConnector}
+ */
+public class OutChannel implements ConnectorChannel {
+
+ private OutputStream outputStream;
+
+ private WritableByteChannel writableByteChannel;
+
+ private HashMap<String, Object> contextAttributeMap = new HashMap();
+
+ public OutChannel(OutputStream outputStream) {
+ this.outputStream = outputStream;
+ writableByteChannel = Channels.newChannel(outputStream);
+
+ }
+
+ @Override
+ public Channel getChannel() {
+ return writableByteChannel;
+ }
+
+ @Override
+ public void closeChannel() throws IOException {
+ outputStream.close();
+ }
+
+ @Override
+ public void addChannelAttribute(String key, Object value) {
+ contextAttributeMap.put(key, value);
+ }
+
+ @Override
+ public Object getChannelAttribute(String key){
+ return contextAttributeMap.get(key);
+ }
+
+
+}
diff --git a/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/PassthroughMediator.java b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/PassthroughMediator.java
new file mode 100644
index 0000000..36b6801
--- /dev/null
+++ b/core/src/main/java/org/apache/airavata/mft/core/bufferedImpl/PassthroughMediator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.core.bufferedImpl;
+
+import org.apache.airavata.mft.core.api.*;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * This class wire the input stream and output stream based on the Channel type.
+ * If either source or destination channels are FileChannel then wire using
+ * zero copy technique of FileChannel class.
+ * Other wise use DirectByteBuffers to copy data from source to destination.
+ */
+public class PassthroughMediator implements Mediator {
+
+ @Override
+ public void mediate(ConnectorChannel src, ConnectorChannel dst, CompletionCallback callback) {
+ ReadableByteChannel rChannel = (ReadableByteChannel) src.getChannel();
+ WritableByteChannel dChannel = (WritableByteChannel) dst.getChannel();
+ try {
+ if (rChannel instanceof FileChannel) {
+ ChannelUtils.transferTo(dChannel, (FileChannel) src);
+ } else if (dChannel instanceof FileChannel) {
+ ChannelUtils.transferFrom(rChannel, (FileChannel) dst);
+ } else {
+ ChannelUtils.copyData(rChannel, dChannel);
+ }
+
+ // dst.closeChannel();
+
+ Object obj = dst.getChannelAttribute(Constants.CONNECTOR);
+ if (obj != null && obj instanceof SinkConnector) {
+ SinkConnector connector = (SinkConnector) obj;
+ boolean success = connector.verifyUpload(dChannel);
+ if (success) {
+ callback.onComplete("Successfully uploaded", null);
+ } else {
+ String msg = "Upload Failed";
+ ConnectorException connectorException = new ConnectorException(msg, null);
+ callback.onComplete("Upload failed ", connectorException);
+ }
+ }
+
+ } catch (IOException e) {
+ String msg = "Upload Failed";
+ ConnectorException connectorException = new ConnectorException(msg, e);
+ callback.onComplete("Upload failed ", connectorException);
+ } finally {
+ Connector sourceConnector = (Connector) src.getChannelAttribute(Constants.CONNECTOR);
+ Connector sinkConnector = (Connector) dst.getChannelAttribute(Constants.CONNECTOR);
+ try {
+ sourceConnector.closeChannel(rChannel);
+ sinkConnector.closeChannel(dChannel);
+ } catch (Exception ex) {
+ String msg = "Error occurred while closing channels";
+ ConnectorException connectorException = new ConnectorException(msg, ex);
+ callback.onComplete(msg, connectorException);
+ }
+ }
+
+ }
+}
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/Main.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/Main.java
new file mode 100644
index 0000000..42cde42
--- /dev/null
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/Main.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.transport.s3;
+
+import org.apache.airavata.mft.core.api.ConnectorChannel;
+import org.apache.airavata.mft.core.bufferedImpl.AbstractConnector;
+import org.apache.airavata.mft.core.bufferedImpl.ConnectorConfig;
+import org.apache.airavata.mft.core.bufferedImpl.PassthroughMediator;
+
+import java.util.Properties;
+
+/**
+ * For Testing
+ */
+public class Main {
+
+
+
+ public static void main(String[] args) {
+
+ Main main = new Main();
+ main.execute();
+
+ }
+
+ public void execute() {
+ Properties connectorConfProp = new Properties();
+ connectorConfProp.put(S3Constants.ACCESS_KEY, "AKIA2SKSIFUH7QDAROSR");
+ connectorConfProp.put(S3Constants.SECRET_KEY, "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");
+ connectorConfProp.put(S3Constants.REGION, "us-east-2");
+
+ Properties srcProp = new Properties();
+ srcProp.put(S3Constants.BUCKET, "blimpit-test");
+ srcProp.put(S3Constants.REMOTE_FILE, "test.pdf");
+
+ Properties dstProp = new Properties();
+ dstProp.put(S3Constants.BUCKET, "blimpit-test");
+ dstProp.put(S3Constants.REMOTE_FILE, "meemure.pdf");
+
+ try {
+
+ ConnectorConfig connectorConfig = new ConnectorConfig(connectorConfProp);
+
+ AbstractConnector s3SourceConnector = new S3SourceConnector();
+ s3SourceConnector.initiate(connectorConfig);
+ ConnectorChannel srcChannel = s3SourceConnector.openChannel(srcProp);
+
+ AbstractConnector s3SinkConnector = new S3SinkConnector();
+ s3SinkConnector.initiate(connectorConfig);
+ ConnectorChannel dstChannel = s3SinkConnector.openChannel(dstProp);
+
+ PassthroughMediator passthroughMediator = new PassthroughMediator();
+
+
+ // Runnable r = () -> {
+ passthroughMediator.mediate(srcChannel, dstChannel, (message, error) -> {
+ try {
+ System.out.println(message);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ });
+ // };
+
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+
+}
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Constants.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Constants.java
new file mode 100644
index 0000000..8058ca0
--- /dev/null
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Constants.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.transport.s3;
+
+/**
+ * Includes constants related to S3 SDK
+ */
+public interface S3Constants {
+
+ String ACCESS_KEY="ACCESS_KEY";
+ String SECRET_KEY="SECRET_KEY";
+ String BUCKET="BUCKET";
+ String REGION="REGION";
+ String REMOTE_FILE="REMOTE_FILE";
+ int CONNECTION_EXPIRE_TIME = 1000 * 60 * 60;
+ String HTTP_CONNECTION = "HTTP_CONNECTION";
+ int HTTP_SUCCESS_RESPONSE_CODE = 200;
+}
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SinkConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SinkConnector.java
new file mode 100644
index 0000000..c2cb503
--- /dev/null
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SinkConnector.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.transport.s3;
+
+import com.amazonaws.HttpMethod;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import org.apache.airavata.mft.core.api.ConnectorChannel;
+import org.apache.airavata.mft.core.api.SinkConnector;
+import org.apache.airavata.mft.core.bufferedImpl.AbstractConnector;
+import org.apache.airavata.mft.core.bufferedImpl.ConnectorConfig;
+import org.apache.airavata.mft.core.bufferedImpl.Constants;
+import org.apache.airavata.mft.core.bufferedImpl.OutChannel;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.channels.Channel;
+import java.util.Properties;
+
+/**
+ * Connector class which connects to a given S3 destination and provides
+ * S3 output Stream.
+ */
+public class S3SinkConnector extends AbstractConnector implements SinkConnector {
+ private AmazonS3 s3Client;
+
+
+ @Override
+ public boolean initiate(ConnectorConfig connectorConfig) {
+ s3Client = S3TransportUtil.getS3Client(connectorConfig.getValue(S3Constants.ACCESS_KEY),
+ connectorConfig.getValue(S3Constants.SECRET_KEY), connectorConfig.getValue(S3Constants.REGION));
+ return true;
+ }
+
+ @Override
+ public ConnectorChannel openChannel(Properties properties) throws IOException {
+ java.util.Date expiration = new java.util.Date();
+ long expTimeMillis = expiration.getTime();
+ expTimeMillis += S3Constants.CONNECTION_EXPIRE_TIME;
+ expiration.setTime(expTimeMillis);
+
+ GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest
+ (properties.getProperty(S3Constants.BUCKET), properties.getProperty(S3Constants.REMOTE_FILE))
+ .withMethod(HttpMethod.PUT)
+ .withExpiration(expiration);
+ URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setDoOutput(true);
+ connection.setRequestMethod("PUT");
+ OutputStream outputStream = connection.getOutputStream();
+ OutChannel outChannel = new OutChannel(outputStream);
+ outChannel.addChannelAttribute(S3Constants.HTTP_CONNECTION, connection);
+ outChannel.addChannelAttribute(Constants.CONNECTOR, this);
+ cacheChannel(outChannel.getChannel(), outChannel);
+ return outChannel;
+ }
+
+ @Override
+ public boolean verifyUpload(Channel channel) {
+ OutChannel outChannel = (OutChannel) getConnectorChannel(channel);
+ HttpURLConnection connection = (HttpURLConnection) outChannel.getChannelAttribute(S3Constants.HTTP_CONNECTION);
+ try {
+ if (connection.getResponseCode() == S3Constants.HTTP_SUCCESS_RESPONSE_CODE) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (Exception error) {
+ return false;
+ }
+ }
+
+
+}
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SourceConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SourceConnector.java
new file mode 100644
index 0000000..ce66c0c
--- /dev/null
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3SourceConnector.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.mft.transport.s3;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3Object;
+import org.apache.airavata.mft.core.api.ConnectorChannel;
+import org.apache.airavata.mft.core.api.SourceConnector;
+import org.apache.airavata.mft.core.bufferedImpl.AbstractConnector;
+import org.apache.airavata.mft.core.bufferedImpl.ConnectorConfig;
+import org.apache.airavata.mft.core.bufferedImpl.Constants;
+import org.apache.airavata.mft.core.bufferedImpl.InChannel;
+
+import java.io.InputStream;
+import java.util.Properties;
+
+/**
+ * Connector class which connects to a given S3 source and provides
+ * S3 Input Stream.
+ */
+public class S3SourceConnector extends AbstractConnector implements SourceConnector {
+
+ private AmazonS3 s3Client;
+
+ @Override
+ public boolean initiate(ConnectorConfig connectorConfig) {
+ s3Client = S3TransportUtil.getS3Client(connectorConfig.getValue(S3Constants.ACCESS_KEY),
+ connectorConfig.getValue(S3Constants.SECRET_KEY), connectorConfig.getValue(S3Constants.REGION));
+ return true;
+ }
+
+ @Override
+ public ConnectorChannel openChannel(Properties properties) {
+ S3Object s3object = s3Client.getObject(properties.getProperty(S3Constants.BUCKET),
+ properties.getProperty(S3Constants.REMOTE_FILE));
+ InputStream inputStream;
+ if (s3object != null && s3object.getObjectContent() != null) {
+ inputStream = s3object.getObjectContent();
+ InChannel inChannel = new InChannel(inputStream);
+ inChannel.addChannelAttribute(Constants.CONNECTOR, this);
+ cacheChannel(inChannel.getChannel(), inChannel);
+ return inChannel;
+ }
+ return null;
+ }
+
+
+}